diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core index 0a0e8addf92e..55f3c7bc1d87 160000 --- a/3rdparty/dmlc-core +++ b/3rdparty/dmlc-core @@ -1 +1 @@ -Subproject commit 0a0e8addf92e1287fd7a25c6314016b8c0138dee +Subproject commit 55f3c7bc1d875fbc7d34fc26651bb8c6818c8355 diff --git a/3rdparty/mkldnn b/3rdparty/mkldnn index a7c5f53832ac..722901c9aaef 160000 --- a/3rdparty/mkldnn +++ b/3rdparty/mkldnn @@ -1 +1 @@ -Subproject commit a7c5f53832acabade6e5086e72c960adedb3c38a +Subproject commit 722901c9aaefa579698df778d061d4848ab8c3e3 diff --git a/3rdparty/mshadow b/3rdparty/mshadow index 3dc80815d965..95ebe0f109ae 160000 --- a/3rdparty/mshadow +++ b/3rdparty/mshadow @@ -1 +1 @@ -Subproject commit 3dc80815d965b56b9a975dc27229361955bf66fe +Subproject commit 95ebe0f109ae021d0d66e2a627ccfc55c3253b55 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f974e8b987c..d8ef524bb389 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -294,8 +294,8 @@ else() add_definitions(-DMXNET_USE_NCCL=0) endif() +include(cmake/ChooseBlas.cmake) if(USE_CUDA AND FIRST_CUDA) - include(cmake/ChooseBlas.cmake) include(3rdparty/mshadow/cmake/Utils.cmake) include(cmake/FirstClassLangCuda.cmake) include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 9b145503ecc1..7a50f158a011 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Contributors of Apache MXNet (incubating) ========================================= MXNet has been developed by a community of people who are interested in large-scale machine learning and deep learning. @@ -196,6 +213,8 @@ List of Contributors * [Piyush Ghai](https://github.com/piyushghai) * [Zach Boldyga](https://github.com/zboldyga) * [Neo Chien](https://github.com/cchung100m) +* [Gordon Reid](https://github.com/gordon1992) +* [Ming Yang](http://ufoym.com) Label Bot --------- diff --git a/KEYS b/KEYS index 03b3ed40ccf6..7b78ea97e3ea 100644 --- a/KEYS +++ b/KEYS @@ -599,3 +599,93 @@ a4LYL628Ksuv1Yxn/Uhb5nDPxU5RKRDeogn07wtaqSG12T9pcIwmExW1RN5kvXc2 3Os= =XL0V -----END PGP PUBLIC KEY BLOCK----- + +pub rsa2048 2019-02-15 [SC] [expires: 2021-02-14] + 0812952358B12DC30536E7E0C06916C3AB88ABFE +uid [ultimate] Qing Lan +sig 3 C06916C3AB88ABFE 2019-02-15 Qing Lan +sub rsa2048 2019-02-15 [E] [expires: 2021-02-14] +sig C06916C3AB88ABFE 2019-02-15 Qing Lan + +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFwb8GEBCADjmi5ZXihPfPbLxhNpbD4HVdT7xGEQgfgkeTA4TdFyBP81zM0F +dTHPQOhHzPGkHrVwUPt3ir2hS46q3L8wni3VRkUU8KPbqbS/a7Wl7LHFFS0lU36J +3uQLElZOFITlaL1dl7cIv+c8xCfmOPlmEtNAPIB26sIM5qzc5l4xvNf1H0Oq0wo6 +VKCsYb4el4nys2U3UBYVQjGyBEwwemHQmFPKg6a2bc/2UhWn4Z+//g0hzIpYtT/S +jua16r5SHy6BUtFGWfU6LQIwmxqc0TNqRdkDU0QY0A+nT6cgx6ghp/qxoLOhgken +Uw8rutC/oFg2VzS9yVsJNrkbQq5Fl/Mz/wqNABEBAAG0HWxhbmtpbmcgPGxhbmtp +bmdAbGFua2luZy5uZXQ+iQFUBBMBCAA+FiEECukpOOn0VXTzKWFpZ3gsZdQlyvsF +Alwb8GECGwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQZ3gsZdQl +yvtmiAgAnFxzWwPSoHZW9U8ejNtLXxqKmfrUVHDJKVVE0zZPEs9kEk95YZ99jqUS +LagTZwdQOGCJe6Fc9XESW5KdzvnwQyJKwGPCIYDVA/CRuGDbs+tY6HCZ7V1DMGYm +EJsQEzdqWKaNLEVNk1LB3uizEjTS3BIVaVZ0+uovv2joM1mgHX9obGZAZRpBm7zG +TsDwQMuAaWpgnh+BQ9b8O0m6Mdlqh4lAHJmqLdwIz3rD1SLKst0WenjVtyBE1k5t +gBSVfQeMThhcbPITUTLdhAmc27QrsJdDdsH42CVhIJMW6aQnBnEKE3QB+C6DzpTi +UIKqPg99PXmBc9BvlugrZRM4KYiqOLkBDQRcG/BhAQgAzS6PeCCIwImwqSRRnwta +R/7ZbjhMDbnbiHhBKjy08vt2tR8cRK8X7utVwd96u0b3a6J4k0VzMcDP7wBYiY0x +GDbEVsmeXxiqh3+Dv2R+/CBebbswlMJqG+imEaGAAA1bk8U8RrjxuKAUOlyVjEaG +m9flPKOo33SK7UTujfnpSZhA6s0wzRSMn3/26c6+/9vpo8pnqooiD+K7eMhIUfjB ++hFRDJxrEPg9pj3hCJhXA1aCIMrq8jjkm+8PyDQWdt9TdnHLD+b5X+DKNNBpZsIk +IxNTqxNNLan4JZ+70z5m68Lqy4EEHFoHwhn8IFjxOMiJ+6MuP32UdYmvUD3owc8y +nwARAQABiQE8BBgBCAAmFiEECukpOOn0VXTzKWFpZ3gsZdQlyvsFAlwb8GECGwwF +CQPCZwAACgkQZ3gsZdQlyvvYwQf/ctlGApkvSxmhKgCTPBCrudGDpO3QsEF+bR6y +w9WvyvniCt6t5M2A+QPOCqdBvNaP/4wP1H2XghKr9XZcTBlxAJc6pdwxQ6IH6SD8 +syEDVTp0oiQytuyWeNoTM9bOFsliCkjiAVphTDTuVrNEgqdnTMo2cDZgry6gwD0Q +1ZOejcb7kIjiThzaKmgpPnnDNkiK2j25fiNhpiIzEVryLKoXvyoj//C/p8lm9KRk +/2bzRm84uoRsUlrr6I5qizakoODkh5+DlzAaes1fv8ED11zzrlIZjh1vBBgDd3QT +jurgAMn1+WYXhnTVUw8+JjWbmZX4NqJJGZyozkF7rgzI3A9CPJkBDQRcLmPJAQgA +6P5AdfBp4QJKfJC96BjDIrXAFXTytD5wxBsyl1WUcq9vnkX6NZGEtTgsuR/F+FBt +Za7fuDRf+ZT5J7sFQbKFJKRvR8shg1dqnIsoztBD0ESC46trD/YZrisDb09JUXN+ +zvha7jUisA7/pVwfenQM8CpnZJiuKIDU5qCTUr7qOjds6g9aCtUSOK41xLAn1y9b +hKJko4xtjUQu6hXjRXXKltl1N0X5TQCw3sR105xTTWx9nAgFu7cc1uGGuOPCpSms +FPkXFMz+75ff9eXtnmpM+9ZxE8c0jTN7p3U16nd2spsQOqQd83nKb3AbfmWrUYC6 +lmiXGlyOqf8fXSSj/d1ZWwARAQABtB5RaW5nIExhbiA8bGFua2luZzUyMEBsaXZl +LmNvbT6JAVQEEwEIAD4WIQSEDPQiKrv4QoSvRxUvo3RD/I+Y+gUCXC5jyQIbAwUJ +A8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRAvo3RD/I+Y+vjlB/4h3trK +2wGFn396m/jqLqBGObCrOTQYJVycfKdndTEvzIm5/U1gU0Q2BzP25k3gYQjjYKDB +ZN5ETrvRa+NUHzK9p4L4mEQpyWI1WR7jJ19Qg3wfjXW2lOJ8hVvGDILKm7JHEOFb +xf8bUsIftAJ+eDs0dFkDEZT+VLGIXWghd236oUy/uYKL0nLofyFZL3IHiBRZkZQf +6nW+wuYlNXJ+Gy8ATUOKmqbhKScOYLT88nvd6XK8ht3agvATTiEZNa7uWeSOZqp/ +Sx10jUKvKif1L3KiRADb461zQdijx1VOpj3OZPKwCViag14TGXot/YsjAHLCROPg +RH7Ey+WHqMzrkI/EuQENBFwuY8kBCADG74gBceYTyUZbwP6y1xKElXacZ4r2aovT +xoyEXGGovkVjR30GT4XWpKBbqFk5S+QtRnJJGCMeVeKbruZc2Jv5cNe5WL8t20SU +shkBi89h2ZWOgw5c8h+5axjyJfPzWuq0PoXZZ0poFVuSojEFuQKotBWqalri3SKu +E7dUlPQ1JzCj60dX03Lqb2fZZEuwNTf3VwXZZHCgRKw0jhvlr0yfZVcIx4C1Vizo +rkmgZoKbTTkcjBwG4xU+QAwtZqtqGUcZsGwGRD/VPYoDwVknTWJN9JwGasM2gdtG +y0IvhjHfhahZXPyiDsUpkvHyrjaKUwF9qkNLlnmV0LxRJYfNYkk1ABEBAAGJATwE +GAEIACYWIQSEDPQiKrv4QoSvRxUvo3RD/I+Y+gUCXC5jyQIbDAUJA8JnAAAKCRAv +o3RD/I+Y+ql8CADZJxKXAEhBN6ao0+OE2RM/+IPUgIuW/B33KxYN/SewVaUeZvIe +Nu3TWLiFa4eTOjc2hNZxRmQRtpmX7faJaynuz/bsfdtdsscuWxNUUiF3Tdd9/Eon +eFaywH8as9uFwYLRmfdb5cvst97XofQOJBEfolOgyGOEEuEVxs6bFa43/SSYCliE +MjhxubT8NqQmIsHGQ4nltxCbmVZNKi7eAjms+QkWX5bqWMqXF5hJisagXvO1Jr75 +ncoq3Wp+X1b+9x8KDGCVd1F5+d0OksPrnNIQVOS8TMw6bJzXLw4yQni+1ZtYTw8p +9eG+gakLwBnBvqs+7HmuvPNELZX5SsKfwRYzmQENBFxnEhcBCADFLKeuIwb8XnYd +MsAEbQIyueHsON04/XJVvULVHlbpF7X4XyUU2aHxkydFN5yGtJZjE7/BXrPSeoK8 +0qtTblb7ZugUgnyhXaN97RqdD6Lv6kha6EFSylUG9Wyfi1Onr0zlgqQ5TwbaTbLJ +UuzN5d7UIPNFZBrWgvH6AC982M3kMGBWEPRiJgCs/k601oX84g6HbynuxaySts1W +N8mC1TjzVuTRq6Cbfr1RAkNcj3mz66DpZLnHsBUPuQ5u0gJTKZyEGRxtT02eXpMI +61eP+79hnAwqZds9BXwO+3cCx/x3qJRLNDpR4mPq9QqETkqTIQd6JPjWPRh0OQDK +300hDmizABEBAAG0HVFpbmcgTGFuIDxsYW5raW5nQGFwYWNoZS5vcmc+iQFUBBMB +CAA+FiEECBKVI1ixLcMFNufgwGkWw6uIq/4FAlxnEhcCGwMFCQPCZwAFCwkIBwIG +FQoJCAsCBBYCAwECHgECF4AACgkQwGkWw6uIq/5KDgf/S8XUCzm+aVJUAcx9BuQV +SzvG8C1Bf/XrYXcm3NxEOMejdkt8FAnaPt9dZM3U8CaMu1a90iiuLyvr3aJvb7/K +/WuaK79jFB+zz8bifQANXgwAQPiUjBU/8FNOjiEywaSYwIvy8SMo5lLs2edV4Y8g +J73BU/WID3aH/BK4sclAs3brNY5le3B2GiAXhqCWB567ZTlc6vpf1DBaivVX8J0H +cEBIDRuitSkWrFYGmx4Ddr7JebkgNBctJaZSkquHxxORLWfFnEqzUhSzKwdZUTTO +bfg31btuFHTiEuQFM+jYGzYBkwW5dlJMRKPTFGg712VwvtbeE6ZkF3AQFARaOlPx +MrkBDQRcZxIXAQgAxRPIrPzi2sv7OpOzKyZZItqSPYVncDPIV4FSMqXbcB0MHT+I +g3tQJFmmFo5q53vdyGwzUmU9ft4jxkif+IQS+pvmXn1q3WNm/+t62ylM6S1hV4sW +ZgJAfrmfID19XtdtyNKSdRIWG+h/q/Mi8clT4RO1/jMnimDNw1KHOBeC3TncJjCn +lpkGVHJMw/S7+odOZmKaXIevJiNpkeXqY+A25w92MNEx7mSn2icNIfhODEdk8GJH +P6kbk2c1q+GPeS9wJap5XCASVSfVlsIPFKm/s9O0OUJ+hmQJq5yXDuEjxE5FyFnq +5uWcYlS3OU7JAJMwaPNVShRzy4+3J4e/F02DjwARAQABiQE8BBgBCAAmFiEECBKV +I1ixLcMFNufgwGkWw6uIq/4FAlxnEhcCGwwFCQPCZwAACgkQwGkWw6uIq/5pCAgA +tZYnewxZPDQN/VHHT4yyYXf67SLWoygiQ9epsIpmvQYJFREbvKlVKUxudSx0CZWc +DTaIup6stXPgB33pmWvV3ia1B+2kNcBPuXGoilHBjmG1oMp2/9Y9NjbFwmsZUo4V +TXGEQvhthiZUelBTWiTrCKYYOl7VEthXpB4OoHJB7wZ+PE4aOmZdArK6cT5/fATT +n4aiPNGpG7CDmCNnGMJgNYEEbqe1RQ7B4xwmNmGJvdVJRsTfy5557hZNfIfVkdES +QTXMfTPP627GwzHQXTdAn9CSGW5FkaSHTVTCZhalBHhAFMDg86ZGUxZDYwhf3s6W +44liPzisQFRxRFOwEubvmw== +=dQvb +-----END PGP PUBLIC KEY BLOCK----- diff --git a/MKLDNN_README.md b/MKLDNN_README.md index ecb721e5fffe..214fc83985fb 100644 --- a/MKLDNN_README.md +++ b/MKLDNN_README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Build/Install MXNet with MKL-DNN A better training and inference performance is expected to be achieved on Intel-Architecture CPUs with MXNet built with [Intel MKL-DNN](https://github.com/intel/mkl-dnn) on multiple operating system, including Linux, Windows and MacOS. diff --git a/Makefile b/Makefile index cc7aebc9fb89..5c5e77fe3a58 100644 --- a/Makefile +++ b/Makefile @@ -489,7 +489,7 @@ build/plugin/%_gpu.o: plugin/%.cu $(CXX) -std=c++11 $(CFLAGS) -MM -MT build/plugin/$*_gpu.o $< >build/plugin/$*_gpu.d $(NVCC) -c -o $@ $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" $< -build/plugin/%.o: plugin/%.cc +build/plugin/%.o: plugin/%.cc | mkldnn @mkdir -p $(@D) $(CXX) -std=c++11 -c $(CFLAGS) -MMD -c $< -o $@ @@ -607,11 +607,7 @@ rpkg: fi mkdir -p R-package/inst/include - cp -rf include/* R-package/inst/include - rm R-package/inst/include/dmlc - rm R-package/inst/include/nnvm - cp -rf 3rdparty/dmlc-core/include/* R-package/inst/include/ - cp -rf 3rdparty/tvm/nnvm/include/* R-package/inst/include + cp -rl include/* R-package/inst/include Rscript -e "if(!require(devtools)){install.packages('devtools', repo = 'https://cloud.r-project.org/')}" Rscript -e "if(!require(roxygen2)||packageVersion('roxygen2') < '6.1.1'){install.packages('roxygen2', repo = 'https://cloud.r-project.org/')}" Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cloud.r-project.org/')); install_deps(pkg='R-package', dependencies = TRUE)" @@ -620,7 +616,7 @@ rpkg: R CMD INSTALL R-package Rscript -e "require(mxnet); mxnet:::mxnet.export('R-package'); warnings()" rm R-package/NAMESPACE - Rscript -e "devtools::document('R-package'); warnings()" + Rscript -e "devtools::document('R-package');warnings()" R CMD INSTALL R-package rpkgtest: @@ -652,6 +648,7 @@ rclean: ifneq ($(EXTRA_OPERATORS),) clean: rclean cyclean $(EXTRA_PACKAGES_CLEAN) $(RM) -r build lib bin deps *~ */*~ */*/*~ */*/*/*~ + (cd scala-package && mvn clean) || true cd $(DMLC_CORE); $(MAKE) clean; cd - cd $(PS_PATH); $(MAKE) clean; cd - cd $(NNVM_PATH); $(MAKE) clean; cd - @@ -661,6 +658,7 @@ clean: rclean cyclean $(EXTRA_PACKAGES_CLEAN) else clean: rclean mkldnn_clean cyclean testclean $(EXTRA_PACKAGES_CLEAN) $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ + (cd scala-package && mvn clean) || true cd $(DMLC_CORE); $(MAKE) clean; cd - cd $(PS_PATH); $(MAKE) clean; cd - cd $(NNVM_PATH); $(MAKE) clean; cd - diff --git a/NEWS.md b/NEWS.md index f06cc35d8b0f..1af4b138e600 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + MXNet Change Log ================ diff --git a/README.md b/README.md index f5d40a6f656b..6a8ecdd99bd0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + +

diff --git a/amalgamation/README.md b/amalgamation/README.md index 3faf4d1e18a4..2ecf1626c1e7 100644 --- a/amalgamation/README.md +++ b/amalgamation/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + MXNet Amalgamation ================== This folder contains a amalgamation generation script to generate the entire mxnet library into one file. diff --git a/amalgamation/amalgamation.py b/amalgamation/amalgamation.py index a3c28f7118e9..0a4be02b8ff9 100644 --- a/amalgamation/amalgamation.py +++ b/amalgamation/amalgamation.py @@ -143,7 +143,12 @@ def expand(x, pending, stage): continue path = m.groups()[0] h = path.strip('./') if "../3rdparty/" not in path else path - source = find_source(h, x, stage) + if h.endswith('complex.h') and x.endswith('openblas_config.h'): + source = '' + elif h.startswith('ps/'): + source = '../3rdparty/ps-lite/include/' + h + else: + source = find_source(h, x, stage) if not source: if (h not in blacklist and h not in sysheaders and diff --git a/amalgamation/dmlc-minimum0.cc b/amalgamation/dmlc-minimum0.cc index 87e08d31c4d6..2fe629b20ed3 100644 --- a/amalgamation/dmlc-minimum0.cc +++ b/amalgamation/dmlc-minimum0.cc @@ -29,6 +29,7 @@ #include "../3rdparty/dmlc-core/src/io/local_filesys.cc" #include "../3rdparty/dmlc-core/src/data.cc" #include "../3rdparty/dmlc-core/src/io.cc" +#include "../3rdparty/dmlc-core/src/io/filesys.cc" #include "../3rdparty/dmlc-core/src/recordio.cc" diff --git a/amalgamation/mxnet_predict0.cc b/amalgamation/mxnet_predict0.cc index c4653d30dd3d..a18e28fd57de 100644 --- a/amalgamation/mxnet_predict0.cc +++ b/amalgamation/mxnet_predict0.cc @@ -43,25 +43,31 @@ #define DISABLE_OPENMP 1 #define DMLC_LOG_STACK_TRACE 0 +#include "src/common/utils.cc" #include "src/ndarray/ndarray_function.cc" #include "src/ndarray/ndarray.cc" #include "src/imperative/imperative.cc" +#include "src/imperative/imperative_utils.cc" #include "src/imperative/cached_op.cc" #include "src/engine/engine.cc" #include "src/engine/naive_engine.cc" +#include "src/engine/openmp.cc" #include "src/profiler/profiler.cc" +#include "src/profiler/aggregate_stats.cc" #include "src/executor/graph_executor.cc" #include "src/executor/attach_op_execs_pass.cc" #include "src/executor/attach_op_resource_pass.cc" #include "src/executor/inplace_addto_detect_pass.cc" +#include "src/executor/infer_graph_attr_pass.cc" #include "src/nnvm/legacy_json_util.cc" #include "src/nnvm/legacy_op_util.cc" +#include "src/nnvm/graph_editor.cc" #include "src/operator/operator.cc" #include "src/operator/operator_util.cc" @@ -93,4 +99,5 @@ #include "src/c_api/c_api_symbolic.cc" #include "src/c_api/c_api_ndarray.cc" #include "src/c_api/c_api_error.cc" +#include "src/c_api/c_api_profile.cc" diff --git a/amalgamation/prep_nnvm.sh b/amalgamation/prep_nnvm.sh index 8e30481d9ebf..a8f63b6b4b1a 100755 --- a/amalgamation/prep_nnvm.sh +++ b/amalgamation/prep_nnvm.sh @@ -40,7 +40,6 @@ echo '#define MSHADOW_FORCE_STREAM #include "mshadow/tensor.h" #include "mxnet/base.h" #include "dmlc/json.h" -#include "nnvm/tuple.h" #include "mxnet/tensor_blob.h"' > temp cat nnvm.cc >> temp mv temp ../../../../amalgamation/nnvm.cc diff --git a/ci/README.md b/ci/README.md index f56a6f6a7978..155a0104a125 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Containerized build & test utilities This folder contains scripts and dockerfiles used to build and test MXNet using diff --git a/ci/docker/install/docs_requirements b/ci/docker/install/docs_requirements index 4e3ce3e55e0b..3cfef1e33901 100644 --- a/ci/docker/install/docs_requirements +++ b/ci/docker/install/docs_requirements @@ -1,3 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + beautifulsoup4==4.6.3 breathe==4.10.0 cpplint==1.3.0 diff --git a/ci/docker/install/ubuntu_clojure.sh b/ci/docker/install/ubuntu_clojure.sh index 7a94cccae946..b20e35898552 100755 --- a/ci/docker/install/ubuntu_clojure.sh +++ b/ci/docker/install/ubuntu_clojure.sh @@ -27,3 +27,4 @@ echo 'Installing Clojure...' wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein chmod 775 lein sudo cp lein /usr/local/bin +echo "Y" | sudo lein downgrade 2.8.3 diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index fc903e5c8899..61a4637830da 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -35,9 +35,11 @@ apt-get install -y \ libatlas-base-dev \ libcurl4-openssl-dev \ libjemalloc-dev \ + libhdf5-dev \ liblapack-dev \ libopenblas-dev \ libopencv-dev \ + libturbojpeg \ libzmq3-dev \ ninja-build \ software-properties-common \ @@ -45,6 +47,11 @@ apt-get install -y \ unzip \ wget +# Use libturbojpeg package as it is correctly compiled with -fPIC flag +# https://github.com/HaxeFoundation/hashlink/issues/147 +ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so + + # Note: we specify an exact cmake version to work around a cmake 3.10 CUDA 10 issue. # Reference: https://github.com/clab/dynet/issues/1457 mkdir /opt/cmake && cd /opt/cmake diff --git a/ci/docker/qemu/README.md b/ci/docker/qemu/README.md index 1dcaa5aeb607..c06b34562b57 100644 --- a/ci/docker/qemu/README.md +++ b/ci/docker/qemu/README.md @@ -1 +1,18 @@ + + + + + + + + + + + + + + + + + These are files used in the docker container that runs QEMU diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 63a2a1b43983..de1b7795ce69 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -292,6 +292,7 @@ build_centos7_cpu() { USE_BLAS=openblas \ USE_MKLDNN=0 \ USE_DIST_KVSTORE=1 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -326,6 +327,7 @@ build_centos7_mkldnn() { USE_LAPACK=1 \ USE_LAPACK_PATH=/usr/lib64/liblapack.so \ USE_BLAS=openblas \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -365,6 +367,8 @@ build_ubuntu_cpu_openblas() { USE_BLAS=openblas \ USE_MKLDNN=0 \ USE_DIST_KVSTORE=1 \ + USE_LIBJPEG_TURBO=1 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -380,6 +384,7 @@ build_ubuntu_cpu_mkl() { USE_MKLDNN=0 \ USE_INTEL_PATH=/opt/intel \ USE_DIST_KVSTORE=1 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -510,6 +515,7 @@ build_ubuntu_cpu_clang39_mkldnn() { USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_OPENMP=0 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -526,6 +532,7 @@ build_ubuntu_cpu_clang60_mkldnn() { USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_OPENMP=1 \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -539,6 +546,7 @@ build_ubuntu_cpu_mkldnn() { ENABLE_TESTCOVERAGE=1 \ USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -552,6 +560,7 @@ build_ubuntu_cpu_mkldnn_mkl() { ENABLE_TESTCOVERAGE=1 \ USE_CPP_PACKAGE=1 \ USE_BLAS=mkl \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -635,6 +644,7 @@ build_ubuntu_gpu_mkldnn() { USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=1 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -651,6 +661,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() { USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=0 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -669,6 +680,7 @@ build_ubuntu_gpu_cuda91_cudnn7() { USE_CPP_PACKAGE=1 \ USE_DIST_KVSTORE=1 \ CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ + USE_SIGNAL_HANDLER=1 \ -j$(nproc) } @@ -953,6 +965,7 @@ unittest_ubuntu_cpu_julia() { export PATH="$1/bin:$PATH" export MXNET_HOME='/work/mxnet' export JULIA_DEPOT_PATH='/work/julia-depot' + export INTEGRATION_TEST=1 julia -e 'using InteractiveUtils; versioninfo()' @@ -1020,8 +1033,6 @@ integrationtest_ubuntu_cpu_asan() { set -ex export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 - # We do not want to fail the build on ASAN errors until memory leaks have been addressed. - export ASAN_OPTIONS=exitcode=0 cd /work/mxnet/build/cpp-package/example/ /work/mxnet/cpp-package/example/get_data.sh ./mlp_cpu diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 5986eaf37ddc..cfbf484756e5 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1028,7 +1028,7 @@ def test_unix_r_gpu() { def test_unix_julia07_cpu() { return ['Julia 0.7: CPU': { node(NODE_LINUX_CPU) { - ws('workspace/ut-julia07-cpu') { + ws('workspace/ut-it-julia07-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia07', false) @@ -1041,7 +1041,7 @@ def test_unix_julia07_cpu() { def test_unix_julia10_cpu() { return ['Julia 1.0: CPU': { node(NODE_LINUX_CPU) { - ws('workspace/ut-julia10-cpu') { + ws('workspace/ut-it-julia10-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia10', false) diff --git a/ci/jenkins/Jenkinsfile_windows_cpu b/ci/jenkins/Jenkinsfile_windows_cpu index 5bc40d625930..01ca673433f6 100644 --- a/ci/jenkins/Jenkinsfile_windows_cpu +++ b/ci/jenkins/Jenkinsfile_windows_cpu @@ -34,14 +34,14 @@ utils.assign_node_labels(utility: 'utility', windows_cpu: 'mxnetwindows-cpu') utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ - custom_steps.compile_windows_cpu() + // custom_steps.compile_windows_cpu() ]) utils.parallel_stage('Tests', [ - custom_steps.test_windows_python2_cpu(), - custom_steps.test_windows_python3_cpu(), - custom_steps.test_windows_julia07_cpu(), - custom_steps.test_windows_julia10_cpu() + // custom_steps.test_windows_python2_cpu(), + // custom_steps.test_windows_python3_cpu(), + // custom_steps.test_windows_julia07_cpu(), + // custom_steps.test_windows_julia10_cpu() ]) } , diff --git a/ci/jenkins/Jenkinsfile_windows_gpu b/ci/jenkins/Jenkinsfile_windows_gpu index 2319f25942de..b3447b960b22 100644 --- a/ci/jenkins/Jenkinsfile_windows_gpu +++ b/ci/jenkins/Jenkinsfile_windows_gpu @@ -34,14 +34,14 @@ utils.assign_node_labels(utility: 'utility', windows_cpu: 'mxnetwindows-cpu', wi utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ - custom_steps.compile_windows_gpu(), - custom_steps.compile_windows_gpu_mkldnn() + // custom_steps.compile_windows_gpu(), + // custom_steps.compile_windows_gpu_mkldnn() ]) utils.parallel_stage('Tests', [ - custom_steps.test_windows_python2_gpu(), - custom_steps.test_windows_python3_gpu(), - custom_steps.test_windows_python3_gpu_mkldnn() + // custom_steps.test_windows_python2_gpu(), + // custom_steps.test_windows_python3_gpu(), + // custom_steps.test_windows_python3_gpu_mkldnn() ]) } , diff --git a/ci/publish/README.md b/ci/publish/README.md index f1ece6f84097..cdd70ce82258 100644 --- a/ci/publish/README.md +++ b/ci/publish/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Publish Settings This folder contains the configuration for restricted nodes on Jenkins for the publishing MXNet artifacts. It also contains a folder called `scala` that contains everything required for publishing to Maven. In this `README`, we provide a brief walkthrough of the Jenkins configuration as well as the usage of the Scala deployment files. Python publishing is TBD. diff --git a/ci/qemu/README.md b/ci/qemu/README.md index 6dde8916b28c..498f8b7a8739 100644 --- a/ci/qemu/README.md +++ b/ci/qemu/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # QEMU base image creation This folder contains scripts and configuration to create a QEMU virtual drive with a debian system. diff --git a/contrib/clojure-package/README.md b/contrib/clojure-package/README.md index 840c15e75033..7566ade66ce8 100644 --- a/contrib/clojure-package/README.md +++ b/contrib/clojure-package/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Clojure MXNet A Clojure Package Built on the MXNet Deep Learning Library diff --git a/contrib/clojure-package/examples/captcha/README.md b/contrib/clojure-package/examples/captcha/README.md index 6b593b2f1c65..be71e07b2120 100644 --- a/contrib/clojure-package/examples/captcha/README.md +++ b/contrib/clojure-package/examples/captcha/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Captcha This is the clojure version of [captcha recognition](https://github.com/xlvector/learning-dl/tree/master/mxnet/ocr) diff --git a/contrib/clojure-package/examples/cnn-text-classification/README.md b/contrib/clojure-package/examples/cnn-text-classification/README.md index 19bb9137334b..f2ed939bee16 100644 --- a/contrib/clojure-package/examples/cnn-text-classification/README.md +++ b/contrib/clojure-package/examples/cnn-text-classification/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # cnn-text-classification An example of text classification using CNN diff --git a/contrib/clojure-package/examples/gan/README.md b/contrib/clojure-package/examples/gan/README.md index 2b46a6cf3e83..9c4bea8ed586 100644 --- a/contrib/clojure-package/examples/gan/README.md +++ b/contrib/clojure-package/examples/gan/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # gan This is an example of how to do a GAN with the MNIST data diff --git a/contrib/clojure-package/examples/imclassification/README.md b/contrib/clojure-package/examples/imclassification/README.md index 4677f289d7e0..06ae9461816e 100644 --- a/contrib/clojure-package/examples/imclassification/README.md +++ b/contrib/clojure-package/examples/imclassification/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # imclassification This shows off how to do image classification with the module api diff --git a/contrib/clojure-package/examples/infer/imageclassifier/README.md b/contrib/clojure-package/examples/infer/imageclassifier/README.md index a8328607c9a2..ef82feebd53f 100644 --- a/contrib/clojure-package/examples/infer/imageclassifier/README.md +++ b/contrib/clojure-package/examples/infer/imageclassifier/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # imageclassifier Run image classification using clojure infer package. diff --git a/contrib/clojure-package/examples/infer/objectdetector/README.md b/contrib/clojure-package/examples/infer/objectdetector/README.md index ec092a296a96..3a38e7d3907e 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/README.md +++ b/contrib/clojure-package/examples/infer/objectdetector/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # objectdetector Run object detection on images using clojure infer package. diff --git a/contrib/clojure-package/examples/infer/predictor/README.md b/contrib/clojure-package/examples/infer/predictor/README.md index 9ca71cf469a0..9a844c4b9353 100644 --- a/contrib/clojure-package/examples/infer/predictor/README.md +++ b/contrib/clojure-package/examples/infer/predictor/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # predictor Run model prediction using clojure infer package. diff --git a/contrib/clojure-package/examples/module/README.md b/contrib/clojure-package/examples/module/README.md index 1b08a52f7db3..fcd16b74a173 100644 --- a/contrib/clojure-package/examples/module/README.md +++ b/contrib/clojure-package/examples/module/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## Instructions This shows off how to use the module api. diff --git a/contrib/clojure-package/examples/multi-label/README.md b/contrib/clojure-package/examples/multi-label/README.md index 27a8c1ff01ff..aebb83be985d 100644 --- a/contrib/clojure-package/examples/multi-label/README.md +++ b/contrib/clojure-package/examples/multi-label/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # multi-label This is a quick example of doing multi-label classification. diff --git a/contrib/clojure-package/examples/neural-style/README.md b/contrib/clojure-package/examples/neural-style/README.md index e05c31552dcd..209e56ae20d2 100644 --- a/contrib/clojure-package/examples/neural-style/README.md +++ b/contrib/clojure-package/examples/neural-style/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # neural-style An example of neural style transfer diff --git a/contrib/clojure-package/examples/pre-trained-models/README.md b/contrib/clojure-package/examples/pre-trained-models/README.md index b0996da69a10..e1aefbca0c5b 100644 --- a/contrib/clojure-package/examples/pre-trained-models/README.md +++ b/contrib/clojure-package/examples/pre-trained-models/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # pre-trained-models This shows examples of how to use the pretrained models. MXNet comes with a number of pretrained models diff --git a/contrib/clojure-package/examples/profiler/README.md b/contrib/clojure-package/examples/profiler/README.md index d8a98d35dbe3..8ab37d713c09 100644 --- a/contrib/clojure-package/examples/profiler/README.md +++ b/contrib/clojure-package/examples/profiler/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # profiler An example of using the profiler. diff --git a/contrib/clojure-package/examples/rnn/README.md b/contrib/clojure-package/examples/rnn/README.md index cad3909447b1..640b63d43f10 100644 --- a/contrib/clojure-package/examples/rnn/README.md +++ b/contrib/clojure-package/examples/rnn/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # rnn diff --git a/contrib/clojure-package/examples/tutorial/README.md b/contrib/clojure-package/examples/tutorial/README.md index cd19eb232760..54f05b36f800 100644 --- a/contrib/clojure-package/examples/tutorial/README.md +++ b/contrib/clojure-package/examples/tutorial/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # tutorial Tutorials are based on the Scala api examples here https://mxnet.incubator.apache.org/api/scala/ndarray.html diff --git a/contrib/clojure-package/examples/visualization/README.md b/contrib/clojure-package/examples/visualization/README.md index 8c6e2c2b3b79..0638d9c49c5d 100644 --- a/contrib/clojure-package/examples/visualization/README.md +++ b/contrib/clojure-package/examples/visualization/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # visualization Run `lein run` to have a sample network visualization printed for you diff --git a/contrib/clojure-package/integration-tests.sh b/contrib/clojure-package/integration-tests.sh index ce480a507f92..3f80ea5fb29a 100755 --- a/contrib/clojure-package/integration-tests.sh +++ b/contrib/clojure-package/integration-tests.sh @@ -26,7 +26,7 @@ lein install # then run through the examples EXAMPLES_HOME=${MXNET_HOME}/contrib/clojure-package/examples # use AWK pattern for blacklisting -TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|dontselect2/'` +TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|cnn-text-classification/'` for i in $TEST_CASES ; do cd ${i} && lein test -done \ No newline at end of file +done diff --git a/contrib/clojure-package/project.clj b/contrib/clojure-package/project.clj index 61d39e28a1d6..e2b999df84d6 100644 --- a/contrib/clojure-package/project.clj +++ b/contrib/clojure-package/project.clj @@ -36,7 +36,7 @@ [org.apache.logging.log4j/log4j-api "2.8.1"] [org.slf4j/slf4j-log4j12 "1.7.25" :exclusions [org.slf4j/slf4j-api]]] :pedantic? :skip - :plugins [[lein-codox "0.10.3" :exclusions [org.clojure/clojure]] + :plugins [[lein-codox "0.10.6" :exclusions [org.clojure/clojure]] [lein-cloverage "1.0.10" :exclusions [org.clojure/clojure]] [lein-cljfmt "0.5.7"]] :codox {:namespaces [#"^org\.apache\.clojure-mxnet\.(?!gen).*"]} diff --git a/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj b/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj index e2e87ed47e2f..7fb603e500b9 100644 --- a/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj +++ b/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj @@ -16,6 +16,7 @@ ;; (ns org.apache.clojure-mxnet.image + "Image API of Clojure package." (:require [t6.from-scala.core :refer [$ $$] :as $] [org.apache.clojure-mxnet.dtype :as dtype] [org.apache.clojure-mxnet.ndarray :as ndarray] @@ -37,7 +38,18 @@ (s/keys :opt-un [::color-flag ::to-rgb ::output])) (defn decode-image - "Decodes an image from an input stream" + "Decodes an image from an input stream with OpenCV + `input-stream`: `InputStream` - Contains the binary encoded image + `color-flag`: 0 or 1 - Convert decoded image to grayscale (0) or color (1) + `to-rgb`: boolean - Whether to convert decoded image to mxnet's default RGB + format (instead of opencv's default BGR) + `output`: nil or `NDArray` + returns: `NDArray` with dtype uint8 + + Ex: + (decode-image input-stream) + (decode-image input-stream {:color-flag 1}) + (decode-image input-stream {:color-flag 0 :output nd})" ([input-stream {:keys [color-flag to-rgb output] :or {color-flag COLOR to-rgb true output nil} :as opts}] @@ -54,7 +66,19 @@ (s/or :none nil? :some ::to-rgb)) (defn read-image - "Reads an image file and returns an ndarray" + "Reads an image file and returns an ndarray with OpenCV. It returns image in + RGB by default instead of OpenCV's default BGR. + `filename`: string - Name of the image file to be loaded + `color-flag`: 0 or 1 - Convert decoded image to grayscale (0) or color (1) + `to-rgb`: boolean - Whether to convert decoded image to mxnet's default RGB + format (instead of opencv's default BGR) + `output`: nil or `NDArray` + returns: `NDArray` with dtype uint8 + + Ex: + (read-image \"cat.jpg\") + (read-image \"cat.jpg\" {:color-flag 0}) + (read-image \"cat.jpg\" {:color-flag 1 :output nd})" ([filename {:keys [color-flag to-rgb output] :or {color-flag nil to-rgb nil output nil} :as opts}] @@ -74,7 +98,17 @@ (s/def ::optional-int (s/or :none nil? :some int?)) (defn resize-image - "Resizes the image array to (width, height)" + "Resizes the image array to (width, height) + `input`: `NDArray` - source image in NDArray + `w`: int - Width of resized image + `h`: int - Height of resized image + `interpolation`: Interpolation method. Default is INTER_LINEAR + `ouput`: nil or `NDArray` + returns: `NDArray` + + Ex: + (resize-image nd-img 300 300) + (resize-image nd-img 28 28 {:output nd})" ([input w h {:keys [interpolation output] :or {interpolation nil output nil} :as opts}] @@ -88,7 +122,21 @@ (resize-image input w h {}))) (defn apply-border - "Pad image border" + "Pad image border with OpenCV. + `input`: `NDArray` - source image in NDArray + `top`: int - Top margin + `bottom`: int - Bottom margin + `left`: int - Left margin + `right`: int - Right margin + `fill-type`: nil or Filling type - Default BORDER_CONSTANT + `value`: nil or double - Deprecated, use `values` instead + `values`: Fill with value(RGB or gray), up to 4 channels + `output`: nil or `NDArray` + returns: `NDArray` + + Ex: + (apply-border img-nd 1 1 1 1) + (apply-border img-nd 3 3 0 0)" ([input top bottom left right {:keys [fill-type value values output] :or {fill-type nil value nil values nil output nil} @@ -109,7 +157,17 @@ (apply-border input top bottom left right {}))) (defn fixed-crop - "Return a fixed crop of the image" + "Return a fixed crop of the image. + `input`: `NDArray` - Source image in NDArray + `x0`: int - Starting x point + `y0`: int - Starting y point + `w`: int - Width of the image + `h`: int - Height of the image + returns: cropped `NDArray` + + Ex: + (fixed-crop nd-img 0 0 28 28) + (fixed-crop nd-img 10 0 100 300)" [input x0 y0 w h] (util/validate! ::ndarray input "Invalid input array") (util/validate! ::int x0 "Invalid starting x coordinate") @@ -119,7 +177,9 @@ (Image/fixedCrop input x0 y0 w h)) (defn rgb-array? - "Returns whether the ndarray is in the RGB format" + "Returns whether the ndarray is in the RGB format + `input`: `NDArray` - Source image in NDArray + returns: boolean" [input] (util/validate! ::ndarray input "Invalid input array") (let [shape (ndarray/shape-vec input)] @@ -133,7 +193,9 @@ (s/and ::ndarray ::all-bytes ::rgb-array)) (defn to-image - "Convert a NDArray image in RGB format to a real image" + "Convert a NDArray image in RGB format to a real image. + `input`: `NDArray` - Source image in NDArray + returns: `BufferedImage`" [input] (util/validate! ::to-image-ndarray input "Invalid input array") (Image/toImage input)) diff --git a/contrib/clojure-package/testing.md b/contrib/clojure-package/testing.md index 8f87c2a76e81..0e87790686f0 100644 --- a/contrib/clojure-package/testing.md +++ b/contrib/clojure-package/testing.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## Help with Testing If you want to give the repo a spin and help make it stable and ready for prime time that would be awesome. diff --git a/cpp-package/README.md b/cpp-package/README.md index 45941555b21c..165a65eaac6a 100644 --- a/cpp-package/README.md +++ b/cpp-package/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet C++ Package The MXNet C++ Package provides C++ API bindings to the users of MXNet. Currently, these bindings are not available as standalone package. diff --git a/cpp-package/example/README.md b/cpp-package/example/README.md index 724478f3d68f..555316dd1ac3 100644 --- a/cpp-package/example/README.md +++ b/cpp-package/example/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet C++ Package Examples ## Building C++ examples diff --git a/cpp-package/example/alexnet.cpp b/cpp-package/example/alexnet.cpp index 1c0f7130d974..e2083a0dfa0a 100644 --- a/cpp-package/example/alexnet.cpp +++ b/cpp-package/example/alexnet.cpp @@ -319,6 +319,7 @@ int main(int argc, char const *argv[]) { } /*don't foget to release the executor*/ delete exec; + delete opt; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/charRNN.cpp b/cpp-package/example/charRNN.cpp index 54b8eea7af39..8951580067a8 100644 --- a/cpp-package/example/charRNN.cpp +++ b/cpp-package/example/charRNN.cpp @@ -500,6 +500,9 @@ void train(const std::string file, int batch_size, int max_epoch, int start_epoc std::string filepath = prefix + "-" + std::to_string(epoch) + ".params"; SaveCheckpoint(filepath, RNN, exe); } + + delete exe; + delete opt; } /*The original example, rnn_cell_demo.py, uses default Xavier as initalizer, which relies on @@ -577,6 +580,9 @@ void trainWithBuiltInRNNOp(const std::string file, int batch_size, int max_epoch std::string filepath = prefix + "-" + std::to_string(epoch) + ".params"; SaveCheckpoint(filepath, RNN, exe); } + + delete exe; + delete opt; } void predict(std::wstring* ptext, int sequence_length, const std::string param_file, @@ -639,6 +645,8 @@ void predict(std::wstring* ptext, int sequence_length, const std::string param_f next = charIndices[n]; ptext->push_back(next); } + + delete exe; } void predictWithBuiltInRNNOp(std::wstring* ptext, int sequence_length, const std::string param_file, @@ -693,6 +701,8 @@ void predictWithBuiltInRNNOp(std::wstring* ptext, int sequence_length, const std next = charIndices[n]; ptext->push_back(next); } + + delete exe; } int main(int argc, char** argv) { diff --git a/cpp-package/example/feature_extract/README.md b/cpp-package/example/feature_extract/README.md index 87cfcb4021f0..0b94bef7705f 100644 --- a/cpp-package/example/feature_extract/README.md +++ b/cpp-package/example/feature_extract/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + This example shows how to extract features with a pretrained model. Execute `run.sh` to: diff --git a/cpp-package/example/googlenet.cpp b/cpp-package/example/googlenet.cpp index c8078afd0af4..26ba51027db6 100644 --- a/cpp-package/example/googlenet.cpp +++ b/cpp-package/example/googlenet.cpp @@ -190,6 +190,7 @@ int main(int argc, char const *argv[]) { } delete exec; + delete opt; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/inception_bn.cpp b/cpp-package/example/inception_bn.cpp index 456e0d913475..2073ebe47fbc 100644 --- a/cpp-package/example/inception_bn.cpp +++ b/cpp-package/example/inception_bn.cpp @@ -232,6 +232,7 @@ int main(int argc, char const *argv[]) { LG << "Validation Accuracy: " << val_acc.Get(); } delete exec; + delete opt; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/inference/README.md b/cpp-package/example/inference/README.md index efd2357570c8..865388b549db 100644 --- a/cpp-package/example/inference/README.md +++ b/cpp-package/example/inference/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet C++ Package Inference Workflow Examples ## Building C++ Inference examples diff --git a/cpp-package/example/lenet.cpp b/cpp-package/example/lenet.cpp index 83c659c7082b..42594548130a 100644 --- a/cpp-package/example/lenet.cpp +++ b/cpp-package/example/lenet.cpp @@ -173,6 +173,7 @@ class Lenet { << ", accuracy: " << ValAccuracy(batch_size * 10, lenet); } delete exe; + delete opt; } private: diff --git a/cpp-package/example/lenet_with_mxdataiter.cpp b/cpp-package/example/lenet_with_mxdataiter.cpp index 39550a3e9e65..33110fee3a88 100644 --- a/cpp-package/example/lenet_with_mxdataiter.cpp +++ b/cpp-package/example/lenet_with_mxdataiter.cpp @@ -177,6 +177,7 @@ int main(int argc, char const *argv[]) { } delete exec; + delete opt; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/mlp.cpp b/cpp-package/example/mlp.cpp index cc16f53cf205..c3760fd3c846 100644 --- a/cpp-package/example/mlp.cpp +++ b/cpp-package/example/mlp.cpp @@ -49,7 +49,7 @@ void OutputAccuracy(mx_float* pred, mx_float* target) { std::cout << "Accuracy: " << right / 128.0 << std::endl; } -void MLP() { +void MLP(int max_epoch) { auto sym_x = Symbol::Variable("X"); auto sym_label = Symbol::Variable("label"); @@ -144,7 +144,6 @@ void MLP() { grad_req_type, aux_states); std::cout << "Training" << std::endl; - int max_epoch = 15000; mx_float learning_rate = 0.0001; for (int epoch_num = 0; epoch_num < max_epoch; ++epoch_num) { exe->Forward(true); @@ -173,8 +172,8 @@ void MLP() { } int main(int argc, char** argv) { - MLP(); + int max_epoch = argc > 1 ? strtol(argv[1], NULL, 10) : 15000; + MLP(max_epoch); MXNotifyShutdown(); return 0; } - diff --git a/cpp-package/example/mlp_cpu.cpp b/cpp-package/example/mlp_cpu.cpp index 93eaf0538269..5d46d40e421f 100644 --- a/cpp-package/example/mlp_cpu.cpp +++ b/cpp-package/example/mlp_cpu.cpp @@ -139,6 +139,7 @@ int main(int argc, char** argv) { } delete exec; + delete opt; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/mlp_csv.cpp b/cpp-package/example/mlp_csv.cpp index 43a14c84e6d8..f12b7c17133d 100644 --- a/cpp-package/example/mlp_csv.cpp +++ b/cpp-package/example/mlp_csv.cpp @@ -267,6 +267,7 @@ int main(int argc, char** argv) { } delete exec; + delete opt; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/mlp_gpu.cpp b/cpp-package/example/mlp_gpu.cpp index 0befde8ae4bc..f6060209a51e 100644 --- a/cpp-package/example/mlp_gpu.cpp +++ b/cpp-package/example/mlp_gpu.cpp @@ -155,6 +155,7 @@ int main(int argc, char** argv) { } delete exec; + delete opt; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/resnet.cpp b/cpp-package/example/resnet.cpp index 7c9dd4daa55a..7200bd42d2de 100644 --- a/cpp-package/example/resnet.cpp +++ b/cpp-package/example/resnet.cpp @@ -242,6 +242,7 @@ int main(int argc, char const *argv[]) { LG << "Validation Accuracy: " << val_acc.Get(); } delete exec; + delete opt; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/test_optimizer.cpp b/cpp-package/example/test_optimizer.cpp index 42547ea6c7c9..70190eff5dc6 100644 --- a/cpp-package/example/test_optimizer.cpp +++ b/cpp-package/example/test_optimizer.cpp @@ -30,6 +30,7 @@ int main(int argc, char** argv) { opt = OptimizerRegistry::Find("adam"); int ret = (opt == 0) ? 1 : 0; + delete opt; MXNotifyShutdown(); return ret; } diff --git a/cpp-package/example/test_score.cpp b/cpp-package/example/test_score.cpp index 7e5096abb816..687683f487f8 100644 --- a/cpp-package/example/test_score.cpp +++ b/cpp-package/example/test_score.cpp @@ -156,6 +156,7 @@ int main(int argc, char** argv) { } delete exec; + delete opt; MXNotifyShutdown(); return score >= MIN_SCORE ? 0 : 1; } diff --git a/cpp-package/tests/ci_test.sh b/cpp-package/tests/ci_test.sh index 7abdef481ec9..18fabea7a7f9 100755 --- a/cpp-package/tests/ci_test.sh +++ b/cpp-package/tests/ci_test.sh @@ -25,22 +25,22 @@ ls -l ../../lib/ ./get_data.sh cp ../../build/cpp-package/example/lenet . -./lenet 10 +./lenet 1 cp ../../build/cpp-package/example/alexnet . ./alexnet 1 cp ../../build/cpp-package/example/lenet_with_mxdataiter . -./lenet_with_mxdataiter 5 +./lenet_with_mxdataiter 1 cp ../../build/cpp-package/example/resnet . -./resnet 5 +./resnet 1 cp ../../build/cpp-package/example/inception_bn . -./inception_bn 5 +./inception_bn 1 cp ../../build/cpp-package/example/mlp . -./mlp +./mlp 150 cp ../../build/cpp-package/example/mlp_cpu . ./mlp_cpu diff --git a/dev_menu.py b/dev_menu.py index 7329b446ed34..1fda4c7aabca 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -118,10 +118,13 @@ def create_virtualenv_default(): ('[Local] Python Unit tests', "./py3_venv/bin/nosetests -v tests/python/unittest/" ), - ('[Website and docs build] Will build to docs/_build/html/', - "ci/docker/runtime_functions.sh deploy_docs"), - ('[Docker] sanity_check. Check for linting and code formatting.', - "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check"), + ('[Docker] Website and docs build outputs to "docs/_build/html/"', + "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh deploy_docs"), + ('[Docker] sanity_check. Check for linting and code formatting and licenses.', + [ + "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check", + "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh nightly_test_rat_check", + ]), ('[Docker] Python3 CPU unittests', [ "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas", diff --git a/docker/Dockerfiles/Dockerfile.in.julia b/docker/Dockerfiles/Dockerfile.in.julia index 42422ddbed54..794f958c9c09 100644 --- a/docker/Dockerfiles/Dockerfile.in.julia +++ b/docker/Dockerfiles/Dockerfile.in.julia @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the julia binding COPY install/julia.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.in.lib.cpu b/docker/Dockerfiles/Dockerfile.in.lib.cpu index 002e2d1e4209..c6de40c5cea3 100644 --- a/docker/Dockerfiles/Dockerfile.in.lib.cpu +++ b/docker/Dockerfiles/Dockerfile.in.lib.cpu @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # dockerfile to build libmxnet.so on CPU FROM ubuntu:14.04 diff --git a/docker/Dockerfiles/Dockerfile.in.lib.gpu b/docker/Dockerfiles/Dockerfile.in.lib.gpu index c7c86a2cf8cd..03b920a685ff 100644 --- a/docker/Dockerfiles/Dockerfile.in.lib.gpu +++ b/docker/Dockerfiles/Dockerfile.in.lib.gpu @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # dockerfile to build libmxnet.so on GPU FROM nvidia/cuda:8.0-cudnn5-devel diff --git a/docker/Dockerfiles/Dockerfile.in.perl b/docker/Dockerfiles/Dockerfile.in.perl index 14b4f0194d6e..6e801dcc48c3 100644 --- a/docker/Dockerfiles/Dockerfile.in.perl +++ b/docker/Dockerfiles/Dockerfile.in.perl @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the perl binding COPY install/perl.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.in.python b/docker/Dockerfiles/Dockerfile.in.python index b7979b231d7d..aa8bb2748d7f 100644 --- a/docker/Dockerfiles/Dockerfile.in.python +++ b/docker/Dockerfiles/Dockerfile.in.python @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the python binding COPY install/python.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.in.r-lang b/docker/Dockerfiles/Dockerfile.in.r-lang index 321094ec6c63..33f6f3533c18 100644 --- a/docker/Dockerfiles/Dockerfile.in.r-lang +++ b/docker/Dockerfiles/Dockerfile.in.r-lang @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the r binding COPY install/r.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.in.scala b/docker/Dockerfiles/Dockerfile.in.scala index 92e098384e13..ee1b96240a25 100644 --- a/docker/Dockerfiles/Dockerfile.in.scala +++ b/docker/Dockerfiles/Dockerfile.in.scala @@ -1,4 +1,23 @@ # -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# # part of the dockerfile to install the scala binding COPY install/scala.sh install/ diff --git a/docker/Dockerfiles/Dockerfile.tensorrt b/docker/Dockerfiles/Dockerfile.tensorrt index 2e95a7618360..f5603e7c1e59 100644 --- a/docker/Dockerfiles/Dockerfile.tensorrt +++ b/docker/Dockerfiles/Dockerfile.tensorrt @@ -1,4 +1,5 @@ # -*- mode: dockerfile -*- +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,6 +17,7 @@ # specific language governing permissions and limitations # under the License. # +# # Dockerfile to run MXNet with TensorRT Integration. FROM nvidia/cuda:9.0-cudnn7-devel diff --git a/docker/Dockerfiles/License.md b/docker/Dockerfiles/License.md deleted file mode 100644 index 614c582aefa8..000000000000 --- a/docker/Dockerfiles/License.md +++ /dev/null @@ -1,20 +0,0 @@ -## Dockerfiles License - -### All Dockerfiles in this directory are provided under an Apache 2.0 license. - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 971a2de7894c..3aa50eccd89d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,3 +1,22 @@ + + # Docker images for MXNET ## How to use diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index f806d5d6f459..6d17c50439ae 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -1,38 +1,57 @@ + + # Release Python Docker Images for MXNet -The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. +The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. It uses the appropriate pip binaries to build different docker images as - * cpu * cpu_mkl * latest (same as cpu) -* gpu_cu90 +* gpu_cu90 * gpu_cu90_mkl * gpu (same as gpu_cu90) -* gpu_cu80 +* gpu_cu80 * gpu_cu80_mkl -* gpu_cu92 +* gpu_cu92 * gpu_cu92_mkl -** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. +** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. Refer: https://pypi.org/project/mxnet/ ### Usage `./build_python_dockerfile.sh ` -For example: +For example: `./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet` -** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. +** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. ### Tests run * [test_conv.py](https://github.com/apache/incubator-mxnet/blob/master/tests/python/train/test_conv.py) * [train_mnist.py](https://github.com/apache/incubator-mxnet/blob/master/example/image-classification/train_mnist.py) -* [test_mxnet.py](https://github.com/apache/incubator-mxnet/blob/master/docker/docker-python/test_mxnet.py): This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. +* [test_mxnet.py](https://github.com/apache/incubator-mxnet/blob/master/docker/docker-python/test_mxnet.py): This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. ### Dockerhub Credentials -Dockerhub credentials will be required to push images at the end of this script. +Dockerhub credentials will be required to push images at the end of this script. Credentials can be provided in the following ways: * **Interactive Login:** Run the script as is and it will ask you for credentials interactively. * **Be Already Logged in:** Login to the mxnet dockerhub account before you run the build script and the script will complete build, test and push. diff --git a/docs/Dockerfile b/docs/Dockerfile index 99bb3d5be492..61a1e5f915e2 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,3 +1,23 @@ +# -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# FROM ubuntu:14.04 MAINTAINER Mu Li diff --git a/docs/README.md b/docs/README.md index 80463cc68d54..c0b40362b7e2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Building and Updating MXNet Documentation The website is hosted at http://mxnet.incubator.apache.org/. diff --git a/docs/_static/js/options.js b/docs/_static/js/options.js index f4fde4e1f2ef..ec3977601c92 100644 --- a/docs/_static/js/options.js +++ b/docs/_static/js/options.js @@ -1,8 +1,9 @@ + /*! * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at @@ -18,7 +19,7 @@ */ /* Installation page display functions for install selector */ -var versionSelect = defaultVersion = 'v1.3.1'; +var versionSelect = defaultVersion = 'v1.4.0'; var platformSelect = 'Linux'; var languageSelect = 'Python'; var processorSelect = 'CPU'; diff --git a/docs/_static/mxnet-theme/index.html b/docs/_static/mxnet-theme/index.html index 302b1732233d..34f675853924 100644 --- a/docs/_static/mxnet-theme/index.html +++ b/docs/_static/mxnet-theme/index.html @@ -23,9 +23,9 @@
-

MXNet 1.3.1 Released

-

This release includes bug fixes, performance improvements, and documentation updates.

- Learn More +

MXNet 1.4.0 Released

+

This release introduces the Java Inference API and Julia API, as well as Control Flow Operators, MKLDNN optimizations, and SVRG optimization.

+ Learn More

A 60-minute Gluon Crash Course

diff --git a/docs/_static/mxnet-theme/navbar.html b/docs/_static/mxnet-theme/navbar.html index 50c3debba395..d39582ecccae 100644 --- a/docs/_static/mxnet-theme/navbar.html +++ b/docs/_static/mxnet-theme/navbar.html @@ -23,11 +23,11 @@

  • Python
  • C++
  • Clojure
    • +
  • Java
  • Julia
  • Perl
  • R
  • Scala
    • -
  • Java
    • @@ -77,11 +77,11 @@

  • Python
  • C++
  • Clojure
    • +
  • Java
  • Julia
  • Perl
  • R
  • Scala
    • -
  • Java
  • diff --git a/docs/api/c++/index.md b/docs/api/c++/index.md index 6bb7a410b341..e09c16e60b28 100644 --- a/docs/api/c++/index.md +++ b/docs/api/c++/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet - C++ API The MXNet C++ Package provides C++ API bindings to the users of MXNet. Currently, these bindings are not available as standalone package. diff --git a/docs/api/clojure/index.md b/docs/api/clojure/index.md index 32abbe06ad79..38e1c5ff9d1e 100644 --- a/docs/api/clojure/index.md +++ b/docs/api/clojure/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet - Clojure API MXNet supports the Clojure programming language. The MXNet Clojure package brings flexible and efficient GPU diff --git a/docs/api/clojure/kvstore.md b/docs/api/clojure/kvstore.md index 25943c365f94..36af3a8e6349 100644 --- a/docs/api/clojure/kvstore.md +++ b/docs/api/clojure/kvstore.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # KVStore API Topics: diff --git a/docs/api/clojure/module.md b/docs/api/clojure/module.md index 88a5308a7cc2..e1414adeb674 100644 --- a/docs/api/clojure/module.md +++ b/docs/api/clojure/module.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Module API The module API provides an intermediate and high-level interface for performing computation with neural networks in MXNet. Module wraps a Symbol and one or more Executors. It has both a high level and intermediate level API. diff --git a/docs/api/clojure/ndarray.md b/docs/api/clojure/ndarray.md index b0e5c991f7d4..991e458ccc04 100644 --- a/docs/api/clojure/ndarray.md +++ b/docs/api/clojure/ndarray.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # NDArray API diff --git a/docs/api/clojure/symbol.md b/docs/api/clojure/symbol.md index 1ec841f153cf..2457dba6d530 100644 --- a/docs/api/clojure/symbol.md +++ b/docs/api/clojure/symbol.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Clojure Symbolic API Topics: diff --git a/docs/api/clojure/symbol_in_pictures.md b/docs/api/clojure/symbol_in_pictures.md index bde274cb238b..f5a56e8dcc3b 100644 --- a/docs/api/clojure/symbol_in_pictures.md +++ b/docs/api/clojure/symbol_in_pictures.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Symbolic Configuration and Execution in Pictures This topic explains symbolic construction and execution in pictures. diff --git a/docs/api/index.md b/docs/api/index.md index 9e7a58f7778c..a1164fc72477 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet APIs diff --git a/docs/api/java/index.md b/docs/api/java/index.md index b2ae035b19c9..099848cdc01b 100644 --- a/docs/api/java/index.md +++ b/docs/api/java/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet - Java Inference API MXNet supports Java for performing inference on a trained model. The MXNet Java Inference API is an extension of the [Scala Infer API](../../api/scala/infer.html) which provides model loading and inference functionality. diff --git a/docs/api/julia/index.md b/docs/api/julia/index.md index f4dc0e5b7498..8aa884eb2772 100644 --- a/docs/api/julia/index.md +++ b/docs/api/julia/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet - Julia API See the [MXNet Julia Reference Manual](https://media.readthedocs.org/pdf/mxnet-test/latest/mxnet-test.pdf). diff --git a/docs/api/perl/index.md b/docs/api/perl/index.md index 6408b525f00c..2d3d2656d7cd 100644 --- a/docs/api/perl/index.md +++ b/docs/api/perl/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet - Perl API MXNet supports the Perl programming language. The MXNet Perl package brings flexible and efficient GPU diff --git a/docs/api/perl/io.md b/docs/api/perl/io.md index 3310f26aba18..ca3b0f1e09f7 100644 --- a/docs/api/perl/io.md +++ b/docs/api/perl/io.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Data Loading API ## Overview @@ -52,6 +69,7 @@ Then we can call `$mod->fit($nd_iter, num_epoch=>2)` to train `loss` by 2 epochs mx->io->NDArrayIter mx->io->CSVIter mx->io->ImageRecordIter +mx->io->ImageRecordInt8Iter mx->io->ImageRecordUInt8Iter mx->io->MNISTIter mx->recordio->MXRecordIO diff --git a/docs/api/perl/kvstore.md b/docs/api/perl/kvstore.md index 7ff73be06b07..4556f68bd837 100644 --- a/docs/api/perl/kvstore.md +++ b/docs/api/perl/kvstore.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # KVStore API Topics: diff --git a/docs/api/perl/module.md b/docs/api/perl/module.md index 132e333dac9c..7b7054646d62 100644 --- a/docs/api/perl/module.md +++ b/docs/api/perl/module.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Module API ## Overview diff --git a/docs/api/perl/ndarray.md b/docs/api/perl/ndarray.md index c7962b9f896f..2ffef392261d 100644 --- a/docs/api/perl/ndarray.md +++ b/docs/api/perl/ndarray.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # NDArray API ## Overview diff --git a/docs/api/perl/symbol.md b/docs/api/perl/symbol.md index 407e912d6fb5..e3ad4816daa4 100644 --- a/docs/api/perl/symbol.md +++ b/docs/api/perl/symbol.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Perl Symbolic API Topics: diff --git a/docs/api/python/autograd/autograd.md b/docs/api/python/autograd/autograd.md index f20799a44351..1905831b16d9 100644 --- a/docs/api/python/autograd/autograd.md +++ b/docs/api/python/autograd/autograd.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Autograd Package ```eval_rst diff --git a/docs/api/python/callback/callback.md b/docs/api/python/callback/callback.md index 714966678a15..225ffcb9a2fd 100644 --- a/docs/api/python/callback/callback.md +++ b/docs/api/python/callback/callback.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Callback API ```eval_rst diff --git a/docs/api/python/contrib/contrib.md b/docs/api/python/contrib/contrib.md index 66fc39152983..4fe80625766c 100644 --- a/docs/api/python/contrib/contrib.md +++ b/docs/api/python/contrib/contrib.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Contrib Package ## Overview diff --git a/docs/api/python/contrib/onnx.md b/docs/api/python/contrib/onnx.md index f8210ad6a001..5691fb022959 100644 --- a/docs/api/python/contrib/onnx.md +++ b/docs/api/python/contrib/onnx.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # ONNX-MXNet API ## Overview diff --git a/docs/api/python/contrib/svrg_optimization.md b/docs/api/python/contrib/svrg_optimization.md index e6e1c3e23ee3..9c5c3a02c88f 100644 --- a/docs/api/python/contrib/svrg_optimization.md +++ b/docs/api/python/contrib/svrg_optimization.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # SVRG Optimization in Python Module API ## Overview diff --git a/docs/api/python/contrib/text.md b/docs/api/python/contrib/text.md index e5f92bc7ef01..577865f501cb 100644 --- a/docs/api/python/contrib/text.md +++ b/docs/api/python/contrib/text.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Text API ## Overview diff --git a/docs/api/python/executor/executor.md b/docs/api/python/executor/executor.md index a37bfa91d2bf..2cf5062e1ed3 100644 --- a/docs/api/python/executor/executor.md +++ b/docs/api/python/executor/executor.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Executor and Executor Manager The executor and executor manager are internal classes for managing symbolic diff --git a/docs/api/python/gluon/contrib.md b/docs/api/python/gluon/contrib.md index 98f36f882164..790f6b496516 100644 --- a/docs/api/python/gluon/contrib.md +++ b/docs/api/python/gluon/contrib.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon Contrib API ## Overview @@ -37,6 +54,9 @@ In the rest of this document, we list routines provided by the `gluon.contrib` p Identity SparseEmbedding SyncBatchNorm + PixelShuffle1D + PixelShuffle2D + PixelShuffle3D ``` ### Recurrent neural network diff --git a/docs/api/python/gluon/data.md b/docs/api/python/gluon/data.md index 99e54c1f034b..b695e1bc3209 100644 --- a/docs/api/python/gluon/data.md +++ b/docs/api/python/gluon/data.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon Data API ## Overview diff --git a/docs/api/python/gluon/gluon.md b/docs/api/python/gluon/gluon.md index 07c98981a166..c063a71d43ea 100644 --- a/docs/api/python/gluon/gluon.md +++ b/docs/api/python/gluon/gluon.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon Package diff --git a/docs/api/python/gluon/loss.md b/docs/api/python/gluon/loss.md index 948f4983d370..8e8f92c5ee49 100644 --- a/docs/api/python/gluon/loss.md +++ b/docs/api/python/gluon/loss.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon Loss API ## Overview diff --git a/docs/api/python/gluon/model_zoo.md b/docs/api/python/gluon/model_zoo.md index 288cf96f1d09..4672a3b7ad81 100644 --- a/docs/api/python/gluon/model_zoo.md +++ b/docs/api/python/gluon/model_zoo.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon Model Zoo ```eval_rst diff --git a/docs/api/python/gluon/nn.md b/docs/api/python/gluon/nn.md index 25c82f066680..2ec66f10416b 100644 --- a/docs/api/python/gluon/nn.md +++ b/docs/api/python/gluon/nn.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon Neural Network Layers ## Overview diff --git a/docs/api/python/gluon/rnn.md b/docs/api/python/gluon/rnn.md index b558b8479e43..f639479d5db3 100644 --- a/docs/api/python/gluon/rnn.md +++ b/docs/api/python/gluon/rnn.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon Recurrent Neural Network API ## Overview diff --git a/docs/api/python/image/image.md b/docs/api/python/image/image.md index 299c54570b17..91e4a6b3ec1a 100644 --- a/docs/api/python/image/image.md +++ b/docs/api/python/image/image.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Image API ## Overview diff --git a/docs/api/python/index.md b/docs/api/python/index.md index de86aedff691..2e23b162ee77 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet - Python API MXNet provides a comprehensive and flexible Python API to serve a broad community of developers with different levels of experience and wide ranging requirements. In this section, we provide an in-depth discussion of the functionality provided by various MXNet Python packages. @@ -157,6 +174,17 @@ Code examples are placed throughout the API documentation and these can be run a rtc/rtc.md ``` +## Run-Time Feature detection / Library Info + +```eval_rst +.. toctree:: + :maxdepth: 1 + + libinfo/libinfo.md +``` + + + ## Symbol API ```eval_rst diff --git a/docs/api/python/io/io.md b/docs/api/python/io/io.md index a069eae61e71..13a612196eee 100644 --- a/docs/api/python/io/io.md +++ b/docs/api/python/io/io.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Data Loading API ## Overview @@ -58,6 +75,7 @@ A detailed tutorial is available at io.CSVIter io.LibSVMIter io.ImageRecordIter + io.ImageRecordInt8Iter io.ImageRecordUInt8Iter io.MNISTIter recordio.MXRecordIO diff --git a/docs/api/python/kvstore/kvstore.md b/docs/api/python/kvstore/kvstore.md index 9f4c06499663..7858ccd02699 100644 --- a/docs/api/python/kvstore/kvstore.md +++ b/docs/api/python/kvstore/kvstore.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # KVStore API .. note:: Direct interactions with ``KVStore`` are dangerous and not recommended. diff --git a/docs/api/python/libinfo/libinfo.md b/docs/api/python/libinfo/libinfo.md new file mode 100644 index 000000000000..3b8d4998aa5b --- /dev/null +++ b/docs/api/python/libinfo/libinfo.md @@ -0,0 +1,110 @@ + + + + + + + + + + + + + + + + + +# Run-Time Feature detection / Library info + +```eval_rst +.. currentmodule:: mxnet.runtime +``` + +## Overview + +The libinfo functionality allows to check for compile-time features supported by the library. + +### Example usage + +``` +In []: import mxnet as mx + ...: import mxnet.runtime + ...: fs = mx.runtime.Features() + +In []: fs +Out[]: [✖ CUDA, ✖ CUDNN, ✖ NCCL, ✖ CUDA_RTC, ✖ TENSORRT, ✔ CPU_SSE, ✔ CPU_SSE2, ✔ CPU_SSE3, ✔ CPU_SSE4_1, ✔ CPU_SSE4_2, ✖ CPU_SSE4A, ✔ CPU_AVX, ✖ CPU_AVX2, ✖ OPENMP, ✖ SSE, ✔ F16C, ✖ JEMALLOC, ✔ BLAS_OPEN, ✖ BLAS_ATLAS, ✖ BLAS_MKL, ✖ BLAS_APPLE, ✔ LAPACK, ✖ MKLDNN, ✔ OPENCV, ✖ CAFFE, ✖ PROFILER, ✖ DIST_KVSTORE, ✖ CXX14, ✔ SIGNAL_HANDLER, ✔ DEBUG] + +In []: fs.keys() +Out[]: odict_keys(['CUDA', 'CUDNN', 'NCCL', 'CUDA_RTC', 'TENSORRT', 'CPU_SSE', 'CPU_SSE2', 'CPU_SSE3', 'CPU_SSE4_1', 'CPU_SSE4_2', 'CPU_SSE4A', 'CPU_AVX', 'CPU_AVX2', 'OPENMP', 'SSE', 'F16C', 'JEMALLOC', 'BLAS_OPEN', 'BLAS_ATLAS', 'BLAS_MKL', 'BLAS_APPLE', 'LAPACK', 'MKLDNN', 'OPENCV', 'CAFFE', 'PROFILER', 'DIST_KVSTORE', 'CXX14', 'SIGNAL_HANDLER', 'DEBUG']) + +In []: type(fs['CUDA']) +Out[]: mxnet.runtime.Feature + +In []: fs['CUDA'].enabled +Out[]: False + +In []: fs.is_enabled('CPU_SSE') +Out[]: True + +In []: fs.is_enabled('CUDA') +Out[]: False + +In []: features = mx.runtime.feature_list() + +In []: features +Out[]: +[✖ CUDA, + ✖ CUDNN, + ✖ NCCL, + ✖ CUDA_RTC, + ✖ TENSORRT, + ✔ CPU_SSE, + ✔ CPU_SSE2, + ✔ CPU_SSE3, + ✔ CPU_SSE4_1, + ✔ CPU_SSE4_2, + ✖ CPU_SSE4A, + ✔ CPU_AVX, + ✖ CPU_AVX2, + ✖ OPENMP, + ✖ SSE, + ✔ F16C, + ✖ JEMALLOC, + ✔ BLAS_OPEN, + ✖ BLAS_ATLAS, + ✖ BLAS_MKL, + ✖ BLAS_APPLE, + ✔ LAPACK, + ✖ MKLDNN, + ✔ OPENCV, + ✖ CAFFE, + ✖ PROFILER, + ✖ DIST_KVSTORE, + ✖ CXX14, + ✔ SIGNAL_HANDLER, + ✔ DEBUG] + +In []: type(features) +Out[]: list + +In []: type(features[0]) +Out[]: mxnet.runtime.Feature +``` + +```eval_rst +.. autosummary:: + :nosignatures: + +``` + +## API Reference + + + +```eval_rst +.. automodule:: mxnet.runtime + :members: +``` + + diff --git a/docs/api/python/metric/metric.md b/docs/api/python/metric/metric.md index 319647a69491..a3e3b545a56d 100644 --- a/docs/api/python/metric/metric.md +++ b/docs/api/python/metric/metric.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Evaluation Metric API ```eval_rst diff --git a/docs/api/python/model.md b/docs/api/python/model.md index dc36648f7fac..967765740aff 100644 --- a/docs/api/python/model.md +++ b/docs/api/python/model.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Model API The model API provides a simplified way to train neural networks using common best practices. diff --git a/docs/api/python/module/module.md b/docs/api/python/module/module.md index 7a86eccd8d3a..86ffad559897 100644 --- a/docs/api/python/module/module.md +++ b/docs/api/python/module/module.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Module API ```eval_rst diff --git a/docs/api/python/ndarray/contrib.md b/docs/api/python/ndarray/contrib.md index d7c9021b5957..f60e7f141adf 100644 --- a/docs/api/python/ndarray/contrib.md +++ b/docs/api/python/ndarray/contrib.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Contrib NDArray API ```eval_rst diff --git a/docs/api/python/ndarray/linalg.md b/docs/api/python/ndarray/linalg.md index 0a85b48304fa..41436c3ba2d1 100644 --- a/docs/api/python/ndarray/linalg.md +++ b/docs/api/python/ndarray/linalg.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Linear Algebra NDArray API ```eval_rst diff --git a/docs/api/python/ndarray/ndarray.md b/docs/api/python/ndarray/ndarray.md index 2df18c286ba7..3cebffccaaee 100644 --- a/docs/api/python/ndarray/ndarray.md +++ b/docs/api/python/ndarray/ndarray.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # NDArray API ```eval_rst diff --git a/docs/api/python/ndarray/random.md b/docs/api/python/ndarray/random.md index 60c565dd5528..74e7d082df27 100644 --- a/docs/api/python/ndarray/random.md +++ b/docs/api/python/ndarray/random.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Random Distribution Generator NDArray API ```eval_rst diff --git a/docs/api/python/ndarray/sparse.md b/docs/api/python/ndarray/sparse.md index 608a9501a039..71dfaafbf294 100644 --- a/docs/api/python/ndarray/sparse.md +++ b/docs/api/python/ndarray/sparse.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Sparse NDArray API ```eval_rst diff --git a/docs/api/python/optimization/contrib.md b/docs/api/python/optimization/contrib.md index 8fc261f4f052..a431508aa22d 100644 --- a/docs/api/python/optimization/contrib.md +++ b/docs/api/python/optimization/contrib.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Contrib Optimization API ```eval_rst diff --git a/docs/api/python/optimization/optimization.md b/docs/api/python/optimization/optimization.md index fa3547b54a8f..03448123a14f 100644 --- a/docs/api/python/optimization/optimization.md +++ b/docs/api/python/optimization/optimization.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Optimization: initialize and update weights ## Overview diff --git a/docs/api/python/rtc/rtc.md b/docs/api/python/rtc/rtc.md index 88e4ed892722..8f5f4e773efc 100644 --- a/docs/api/python/rtc/rtc.md +++ b/docs/api/python/rtc/rtc.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Run-Time Compilation API ```eval_rst diff --git a/docs/api/python/symbol/contrib.md b/docs/api/python/symbol/contrib.md index a0253216f945..2a6a5efe29be 100644 --- a/docs/api/python/symbol/contrib.md +++ b/docs/api/python/symbol/contrib.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Contrib Symbol API ```eval_rst diff --git a/docs/api/python/symbol/linalg.md b/docs/api/python/symbol/linalg.md index d22ca8e171f9..f1891e29f896 100644 --- a/docs/api/python/symbol/linalg.md +++ b/docs/api/python/symbol/linalg.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Linear Algebra Symbol API ```eval_rst diff --git a/docs/api/python/symbol/random.md b/docs/api/python/symbol/random.md index 1ecaf38830fc..b62747c6ee7b 100644 --- a/docs/api/python/symbol/random.md +++ b/docs/api/python/symbol/random.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Random Distribution Generator Symbol API ```eval_rst diff --git a/docs/api/python/symbol/rnn.md b/docs/api/python/symbol/rnn.md index 722180a27789..287e5a178ad2 100644 --- a/docs/api/python/symbol/rnn.md +++ b/docs/api/python/symbol/rnn.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # RNN Cell Symbol API ```eval_rst diff --git a/docs/api/python/symbol/sparse.md b/docs/api/python/symbol/sparse.md index cd8272cedd7d..473bb4bbe530 100644 --- a/docs/api/python/symbol/sparse.md +++ b/docs/api/python/symbol/sparse.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Sparse Symbol API ```eval_rst diff --git a/docs/api/python/symbol/symbol.md b/docs/api/python/symbol/symbol.md index 0fc2aa7c6cf2..fea746bb02f4 100644 --- a/docs/api/python/symbol/symbol.md +++ b/docs/api/python/symbol/symbol.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Symbol API ```eval_rst @@ -320,6 +337,7 @@ Composite multiple symbols into a new one by an operator. :nosignatures: Symbol.infer_type + Symbol.infer_type_partial Symbol.infer_shape Symbol.infer_shape_partial ``` diff --git a/docs/api/python/symbol_in_pictures/symbol_in_pictures.md b/docs/api/python/symbol_in_pictures/symbol_in_pictures.md index 5ef4395b9af6..23a9757b83d6 100644 --- a/docs/api/python/symbol_in_pictures/symbol_in_pictures.md +++ b/docs/api/python/symbol_in_pictures/symbol_in_pictures.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Symbolic Configuration and Execution in Pictures This topic explains symbolic construction and execution in pictures. diff --git a/docs/api/python/tools/test_utils.md b/docs/api/python/tools/test_utils.md index e29d9b99a844..a442e9b46941 100644 --- a/docs/api/python/tools/test_utils.md +++ b/docs/api/python/tools/test_utils.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Test Utilities This module has a variety of tools that help using and testing MXNet. diff --git a/docs/api/python/tools/visualization.md b/docs/api/python/tools/visualization.md index 6faa24980dd8..c807ce6fbe05 100644 --- a/docs/api/python/tools/visualization.md +++ b/docs/api/python/tools/visualization.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Visualization This module contains visualization features. diff --git a/docs/api/r/index.md b/docs/api/r/index.md index 822c59dbe3cc..e8032cf62bfb 100644 --- a/docs/api/r/index.md +++ b/docs/api/r/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet - R API See the [MXNet R Reference Manual](https://s3.amazonaws.com/mxnet-prod/docs/R/mxnet-r-reference-manual.pdf). diff --git a/docs/api/scala/index.md b/docs/api/scala/index.md index f7a150019872..bda1945b769d 100644 --- a/docs/api/scala/index.md +++ b/docs/api/scala/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet - Scala API MXNet supports the Scala programming language. The MXNet Scala package brings flexible and efficient GPU diff --git a/docs/api/scala/infer.md b/docs/api/scala/infer.md index dfa21c178416..d9a4a62bb7c7 100644 --- a/docs/api/scala/infer.md +++ b/docs/api/scala/infer.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Infer API The MXNet Scala Infer API provides you with model loading and inference functionality using the MXNet Scala package. diff --git a/docs/api/scala/io.md b/docs/api/scala/io.md index ff212d1479ed..9a7a865f9ea5 100644 --- a/docs/api/scala/io.md +++ b/docs/api/scala/io.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Scala Data Loading API This topic introduces the data input method for MXNet. MXNet uses an iterator to provide data to the neural network. Iterators do some preprocessing and generate batches for the neural network. diff --git a/docs/api/scala/kvstore.md b/docs/api/scala/kvstore.md index e195c4d7e720..30ea97d207f6 100644 --- a/docs/api/scala/kvstore.md +++ b/docs/api/scala/kvstore.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # KVStore API Topics: diff --git a/docs/api/scala/model.md b/docs/api/scala/model.md index 3e09d15218b8..12156132d35a 100644 --- a/docs/api/scala/model.md +++ b/docs/api/scala/model.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Scala Model API The model API provides a simplified way to train neural networks using common best practices. diff --git a/docs/api/scala/module.md b/docs/api/scala/module.md index 06487f734a3f..09e7c657c12e 100644 --- a/docs/api/scala/module.md +++ b/docs/api/scala/module.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Module API The module API provides an intermediate and high-level interface for performing computation with neural networks in MXNet. A *module* is an instance of subclasses of the `BaseModule`. The most widely used module class is called `Module`. Module wraps a `Symbol` and one or more `Executors`. For a full list of functions, see `BaseModule`. A subclass of modules might have extra interface functions. This topic provides some examples of common use cases. All of the module APIs are in the `Module` namespace. diff --git a/docs/api/scala/ndarray.md b/docs/api/scala/ndarray.md index 9e87d397c8b8..0c13669d64ab 100644 --- a/docs/api/scala/ndarray.md +++ b/docs/api/scala/ndarray.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # NDArray API diff --git a/docs/api/scala/symbol.md b/docs/api/scala/symbol.md index c10d5fb60a24..aaddc2a8a2f0 100644 --- a/docs/api/scala/symbol.md +++ b/docs/api/scala/symbol.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Scala Symbolic API Topics: diff --git a/docs/api/scala/symbol_in_pictures.md b/docs/api/scala/symbol_in_pictures.md index 0fa12d0a7557..23ece95e835a 100644 --- a/docs/api/scala/symbol_in_pictures.md +++ b/docs/api/scala/symbol_in_pictures.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Symbolic Configuration and Execution in Pictures This topic explains symbolic construction and execution in pictures. diff --git a/docs/architecture/exception_handling.md b/docs/architecture/exception_handling.md index 5b4448a53ab6..6a9ab9ae0c4c 100644 --- a/docs/architecture/exception_handling.md +++ b/docs/architecture/exception_handling.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Exception Handling in MXNet This tutorial explains the exception handling support in MXNet, diff --git a/docs/architecture/index.md b/docs/architecture/index.md index 189e76e62fa5..3c71edfeeb05 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Architecture Building a high-performance deep learning library diff --git a/docs/architecture/note_data_loading.md b/docs/architecture/note_data_loading.md index 7c92c86a9c67..293b67572cb3 100644 --- a/docs/architecture/note_data_loading.md +++ b/docs/architecture/note_data_loading.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Designing Efficient Data Loaders for Deep Learning Data loading is an important component of any machine learning system. diff --git a/docs/architecture/note_engine.md b/docs/architecture/note_engine.md index 237d8d61b77d..6d0e1e19fcef 100644 --- a/docs/architecture/note_engine.md +++ b/docs/architecture/note_engine.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Dependency Engine for Deep Learning We always want deep learning libraries diff --git a/docs/architecture/note_memory.md b/docs/architecture/note_memory.md index 8ec77c395f31..e8b795bbd5c4 100644 --- a/docs/architecture/note_memory.md +++ b/docs/architecture/note_memory.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Optimizing Memory Consumption in Deep Learning Over the last ten years, a constant trend in deep learning diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 6a37f8830479..e2e9ce2bd3eb 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet System Architecture ![System Overview](https://raw.githubusercontent.com/dmlc/dmlc.github.io/master/img/mxnet/system/overview.png) @@ -284,9 +301,9 @@ The `OperatorProperty` interface consists of: * **InferShape:** ```c++ - virtual bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const = 0; + virtual bool InferShape(mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape, + mxnet::ShapeVector *aux_shape) const = 0; ``` This interface has two purposes: @@ -305,9 +322,9 @@ MXNet defines two interfaces to achieve this: ```c++ virtual std::vector ForwardResource( - const std::vector &in_shape) const; + const mxnet::ShapeVector &in_shape) const; virtual std::vector BackwardResource( - const std::vector &in_shape) const; + const mxnet::ShapeVector &in_shape) const; ``` The `ResourceRequest` structure (in `resource.h`) currently contains only a type flag: @@ -456,7 +473,7 @@ To do so, you could define a `ConvolutionParam` structure, as follows: ```c++ #include struct ConvolutionParam : public dmlc::Parameter { - TShape kernel, stride, pad; + mxnet::TShape kernel, stride, pad; uint32_t num_filter, num_group, workspace; bool no_bias; }; @@ -565,10 +582,10 @@ must be provided before any calculation occurs. let's check input data shape consistency and provide output shape. ```cpp - typedef TShape (*UnaryShapeFunction)(const TShape& src, + typedef mxnet::TShape (*UnaryShapeFunction)(const mxnet::TShape& src, const EnvArguments& env); - typedef TShape (*BinaryShapeFunction)(const TShape& lhs, - const TShape& rhs, + typedef mxnet::TShape (*BinaryShapeFunction)(const mxnet::TShape& lhs, + const mxnet::TShape& rhs, const EnvArguments& env); ``` You can use `mshadow::TShape` to check input data shape and designate output data shape. @@ -594,9 +611,9 @@ In our smooth l1 loss example, it's okay to use the default behavior whereby the Written explicitly, it is: ```cpp - inline TShape SmoothL1Shape_(const TShape& src, + inline mxnet::TShape SmoothL1Shape_(const mxnet::TShape& src, const EnvArguments& env) { - return TShape(src); + return mxnet::TShape(src); } ``` diff --git a/docs/architecture/program_model.md b/docs/architecture/program_model.md index 519a9a9024d8..2731a3df7394 100644 --- a/docs/architecture/program_model.md +++ b/docs/architecture/program_model.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Deep Learning Programming Style However much we might ultimately care about performance, diff --git a/docs/architecture/rnn_interface.md b/docs/architecture/rnn_interface.md index dc0b6a7958ef..5cd5a7b85835 100644 --- a/docs/architecture/rnn_interface.md +++ b/docs/architecture/rnn_interface.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Survey of Existing Interfaces and Implementations Commonly used deep learning libraries with good RNN/LSTM support include [Theano](http://deeplearning.net/software/theano/library/scan.html) and its wrappers [Lasagne](http://lasagne.readthedocs.org/en/latest/modules/layers/recurrent.html) and [Keras](http://keras.io/layers/recurrent/); [CNTK](https://cntk.codeplex.com/); [TensorFlow](https://www.tensorflow.org/tutorials/sequences/recurrent); and various implementations in Torch, such as [this well-known character-level language model tutorial](https://github.com/karpathy/char-rnn), [this](https://github.com/Element-Research/rnn). diff --git a/docs/build_version_doc/Dockerfile b/docs/build_version_doc/Dockerfile index 0cc2680ffaef..8bae701199af 100755 --- a/docs/build_version_doc/Dockerfile +++ b/docs/build_version_doc/Dockerfile @@ -1,3 +1,23 @@ +# -*- mode: dockerfile -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# FROM ubuntu:16.04 LABEL maintainer="markhama@amazon.com" diff --git a/docs/build_version_doc/README.md b/docs/build_version_doc/README.md index 50802371cea2..4821f50f5d6e 100644 --- a/docs/build_version_doc/README.md +++ b/docs/build_version_doc/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Documentation Build Scripts This folder contains a variety of scripts to generate the MXNet.io website as well as the docs for different versions of MXNet. diff --git a/docs/build_version_doc/setup_docs_ubuntu.sh b/docs/build_version_doc/setup_docs_ubuntu.sh index 22b2fe2c2f71..baa23f684c09 100755 --- a/docs/build_version_doc/setup_docs_ubuntu.sh +++ b/docs/build_version_doc/setup_docs_ubuntu.sh @@ -64,6 +64,7 @@ echo "Installing Clojure dependencies..." wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein chmod 775 lein sudo cp lein /usr/local/bin +echo "Y" | sudo lein downgrade 2.8.3 echo "Installing R dependencies..." diff --git a/docs/community/contribute.md b/docs/community/contribute.md index 1b83638edcb5..50aa1a607a44 100644 --- a/docs/community/contribute.md +++ b/docs/community/contribute.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Contributing to MXNet Apache MXNet (incubating) is a community led, open source deep learning project. We welcome new members and look forward to your contributions. Here you will find how to get started and links to detailed information on MXNet best practices and processes. diff --git a/docs/community/ecosystem.md b/docs/community/ecosystem.md index 100ed973cf3f..101f76d4c380 100644 --- a/docs/community/ecosystem.md +++ b/docs/community/ecosystem.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet's Ecosystem Community contributions to MXNet have added many new valuable features and functionality to support use cases such as model serving & portability, easy and flexible APIs, and educational material like crash courses and online books. This ecosystem page lists the projects that use MXNet, teach MXNet, or augment MXNet in some way. diff --git a/docs/community/index.md b/docs/community/index.md index 7bdb1c213503..5db4253a65df 100644 --- a/docs/community/index.md +++ b/docs/community/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Community ```eval_rst diff --git a/docs/community/mxnet_channels.md b/docs/community/mxnet_channels.md index 8bc8471dffd1..df05855dfedd 100644 --- a/docs/community/mxnet_channels.md +++ b/docs/community/mxnet_channels.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Join MXNet Development Discussion Converse with the MXNet community via the following channels: diff --git a/docs/community/powered_by.md b/docs/community/powered_by.md index 82748ef7df2a..4b854c4afdc6 100644 --- a/docs/community/powered_by.md +++ b/docs/community/powered_by.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Powered By
    diff --git a/docs/faq/add_op_in_backend.md b/docs/faq/add_op_in_backend.md index c44a0aa05235..15f4ed9fbab4 100644 --- a/docs/faq/add_op_in_backend.md +++ b/docs/faq/add_op_in_backend.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # A Beginner's Guide to Implementing Operators in MXNet Backend ## Introduction @@ -158,8 +175,8 @@ element-wise multiplication and addition. For our `quadratic` operator, shape inference possesses quite similar logic. ```cpp inline bool QuadraticOpShape(const nnvm::NodeAttrs& attrs, - std::vector* in_attrs, - std::vector* out_attrs) { + mxnet::ShapeVector* in_attrs, + mxnet::ShapeVector* out_attrs) { CHECK_EQ(in_attrs->size(), 1U); CHECK_EQ(out_attrs->size(), 1U); @@ -199,8 +216,8 @@ The function `QuadraticOpShape` posted here is for the purpose of illustration o ```cpp template inline bool ElemwiseShape(const nnvm::NodeAttrs& attrs, - std::vector *in_attrs, - std::vector *out_attrs); + mxnet::ShapeVector *in_attrs, + mxnet::ShapeVector *out_attrs); ``` The same logic goes for data type inference. We will leave the analysis of diff --git a/docs/faq/bucketing.md b/docs/faq/bucketing.md index dbfdedde2acf..b5fb987d23e4 100644 --- a/docs/faq/bucketing.md +++ b/docs/faq/bucketing.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Bucketing in MXNet When we train recurrent neural networks (RNNs), we _unroll_ the network in time. For a single example of length T, we would unroll the network T steps. diff --git a/docs/faq/caffe.md b/docs/faq/caffe.md index a9bab3cdb549..e0362ca2776b 100644 --- a/docs/faq/caffe.md +++ b/docs/faq/caffe.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # How to | Convert from Caffe to MXNet Key topics covered include the following: diff --git a/docs/faq/cloud.md b/docs/faq/cloud.md index 67b28f8b4338..ecc0f46e02c0 100644 --- a/docs/faq/cloud.md +++ b/docs/faq/cloud.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet on the Cloud Deep learning can require extremely powerful hardware, often for unpredictable durations of time. diff --git a/docs/faq/develop_and_hack.md b/docs/faq/develop_and_hack.md index da53fd010bb0..0e7d221f7dc3 100644 --- a/docs/faq/develop_and_hack.md +++ b/docs/faq/develop_and_hack.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Develop and Hack MXNet - [Create new operators](new_op.md) - [Use Torch from MXNet](torch.md) diff --git a/docs/faq/distributed_training.md b/docs/faq/distributed_training.md index 8d8666ff066c..4d4a220c5661 100644 --- a/docs/faq/distributed_training.md +++ b/docs/faq/distributed_training.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Distributed Training in MXNet MXNet supports distributed training enabling us to leverage multiple machines for faster training. In this document, we describe how it works, how to launch a distributed training job and diff --git a/docs/faq/env_var.md b/docs/faq/env_var.md index 83368bf4d0c3..f49cb1997691 100644 --- a/docs/faq/env_var.md +++ b/docs/faq/env_var.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Environment Variables ===================== MXNet has several settings that you can change with environment variables. @@ -174,6 +191,12 @@ When USE_PROFILER is enabled in Makefile or CMake, the following environments ca ## Other Environment Variables +* MXNET_GPU_WORKER_NSTREAMS + - Values: 1, or 2 ```(default=1)``` + - Determines the number of GPU streams available to operators for their functions. + - Setting this to 2 may yield a modest performance increase, since ops like the cuDNN convolution op can then calculate their data- and weight-gradients in parallel. + - Setting this to 2 may also increase a model's demand for GPU global memory. + * MXNET_CUDNN_AUTOTUNE_DEFAULT - Values: 0, 1, or 2 ```(default=1)``` - The default value of cudnn auto tuning for convolution layers. diff --git a/docs/faq/faq.md b/docs/faq/faq.md index 668587ec6888..b7866be5c03a 100644 --- a/docs/faq/faq.md +++ b/docs/faq/faq.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Frequently Asked Questions This topic provides answers to the frequently asked questions on [mxnet/issues](https://github.com/dmlc/mxnet/issues). Before posting an issue, please check this page. If you would like to contribute to this page, please make the questions and answers simple. If your answer is extremely detailed, please post it elsewhere and link to it. diff --git a/docs/faq/finetune.md b/docs/faq/finetune.md index 04244d15b0b9..4a03eefa05c9 100644 --- a/docs/faq/finetune.md +++ b/docs/faq/finetune.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Fine-tune with Pretrained Models diff --git a/docs/faq/float16.md b/docs/faq/float16.md index b4cd97b30e5c..323218ce7df6 100644 --- a/docs/faq/float16.md +++ b/docs/faq/float16.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Mixed precision training using float16 In this tutorial you will walk through how one can train deep learning neural networks with mixed precision on supported hardware. You will first see how to use float16 (both with Gluon and Symbolic APIs) and then some techniques on achieving good performance and accuracy. diff --git a/docs/faq/gradient_compression.md b/docs/faq/gradient_compression.md index e2dbd3271d8d..9e926c52feca 100644 --- a/docs/faq/gradient_compression.md +++ b/docs/faq/gradient_compression.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gradient Compression Gradient Compression reduces communication bandwidth, and in some scenarios, it can make training more scalable and efficient without significant loss in convergence rate or accuracy. Example implementations with GPUs, CPUs, and distributed training are provided in this document. diff --git a/docs/faq/index.md b/docs/faq/index.md index fe91f7ca43b7..92f1ccde00f4 100644 --- a/docs/faq/index.md +++ b/docs/faq/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet FAQ ```eval_rst diff --git a/docs/faq/model_parallel_lstm.md b/docs/faq/model_parallel_lstm.md index b78b2c574dcc..e9a8768b6fec 100644 --- a/docs/faq/model_parallel_lstm.md +++ b/docs/faq/model_parallel_lstm.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Training with Multiple GPUs Using Model Parallelism Training deep learning models can be resource intensive. Even with a powerful GPU, some models can take days or weeks to train. diff --git a/docs/faq/multi_devices.md b/docs/faq/multi_devices.md index a43879cb5233..60a8003f75d0 100644 --- a/docs/faq/multi_devices.md +++ b/docs/faq/multi_devices.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Run MXNet on Multiple CPU/GPUs with Data Parallelism _MXNet_ supports training with multiple CPUs and GPUs, which may be located on different physical machines. diff --git a/docs/faq/new_op.md b/docs/faq/new_op.md index 994a2a6f823e..4c10708b944d 100644 --- a/docs/faq/new_op.md +++ b/docs/faq/new_op.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # How to Create New Operators (Layers) This tutorials walks you through the process of creating new MXNet operators (or layers). @@ -241,7 +258,7 @@ can add argument descriptions in bulk with `.add_arguments(ActivationParam::__FI #### FInferShape or TIsBackward (for Backward Only Ops) -Normally operators need to have `FInferShape` with prototype `bool(const nnvm::NodeAttrs& attrs, std::vector *in_attrs, std::vector *out_attrs)`. `FInferShape` fills unknown shapes (`shape.ndim() == 0`) in in_attrs/out_attrs based on known shapes in in_attrs/out_attrs. Use `ElemwiseShape` for simple operators with uniform shapes. +Normally operators need to have `FInferShape` with prototype `bool(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector *in_attrs, mxnet::ShapeVector *out_attrs)`. `FInferShape` fills unknown shapes (`shape.ndim() == 0`) in in_attrs/out_attrs based on known shapes in in_attrs/out_attrs. Use `ElemwiseShape` for simple operators with uniform shapes. Operators that are only used for a backward pass can instead register `.set_attr("TIsBackward", true)` and their shapes with be copied from the corresponding forward operators. diff --git a/docs/faq/nnpack.md b/docs/faq/nnpack.md index ed38cb07df7e..c0b27b9bf4e6 100644 --- a/docs/faq/nnpack.md +++ b/docs/faq/nnpack.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ### NNPACK for Multi-Core CPU Support in MXNet [NNPACK](https://github.com/Maratyszcza/NNPACK) is an acceleration package for neural network computations, which can run on x86-64, ARMv7, or ARM64 architecture CPUs. diff --git a/docs/faq/perf.md b/docs/faq/perf.md index f116ede11d56..00310dfbb5bd 100644 --- a/docs/faq/perf.md +++ b/docs/faq/perf.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Some Tips for Improving MXNet Performance Even after fixing the training or deployment environment and parallelization scheme, a number of configuration settings and data-handling choices can impact the _MXNet_ performance. diff --git a/docs/faq/recordio.md b/docs/faq/recordio.md index 3091052ef6f3..7367b524224d 100644 --- a/docs/faq/recordio.md +++ b/docs/faq/recordio.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## Create a Dataset Using RecordIO RecordIO implements a file format for a sequence of records. We recommend storing images as records and packing them together. The benefits include: diff --git a/docs/faq/s3_integration.md b/docs/faq/s3_integration.md index 024356706339..49520b5814e1 100644 --- a/docs/faq/s3_integration.md +++ b/docs/faq/s3_integration.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Use data from S3 for training AWS S3 is a cloud-based object storage service that allows storage and retrieval of large amounts of data at a very low cost. This makes it an attractive option to store large training datasets. MXNet is deeply integrated with S3 for this purpose. diff --git a/docs/faq/security.md b/docs/faq/security.md index 0615acda3435..88d48be35ecb 100644 --- a/docs/faq/security.md +++ b/docs/faq/security.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Security Best Practices MXNet framework has no built-in security protections. It assumes that the MXNet entities involved in model training and inferencing (hosting) are fully trusted. It also assumes that their communications cannot be eavesdropped or tampered with. MXNet consumers shall ensure that the above assumptions are met. diff --git a/docs/faq/smart_device.md b/docs/faq/smart_device.md index 2584b4c36caf..c56457a626cb 100644 --- a/docs/faq/smart_device.md +++ b/docs/faq/smart_device.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Deep Learning in a Single File for Smart Devices Deep learning (DL) systems are complex and often depend on a number of libraries. diff --git a/docs/faq/visualize_graph.md b/docs/faq/visualize_graph.md index 06010213242c..adeb0e49136f 100644 --- a/docs/faq/visualize_graph.md +++ b/docs/faq/visualize_graph.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # How to visualize Neural Networks as computation graph Here, we'll demonstrate how to use ```mx.viz.plot_network``` diff --git a/docs/faq/why_mxnet.md b/docs/faq/why_mxnet.md old mode 100755 new mode 100644 index ed8cef143070..140e853e2692 --- a/docs/faq/why_mxnet.md +++ b/docs/faq/why_mxnet.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Why MXNet? Probably, if you've stumbled upon this page, you've heard of _deep learning_. diff --git a/docs/gluon/index.md b/docs/gluon/index.md index 96e8e36dbf20..469222e46293 100644 --- a/docs/gluon/index.md +++ b/docs/gluon/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # About Gluon ![gluon logo](https://github.com/dmlc/web-data/blob/master/mxnet/image/image-gluon-logo.png?raw=true) diff --git a/docs/index.md b/docs/index.md index ab6a95dc0ddd..e7f314eb169b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet ```eval_rst diff --git a/docs/install/amazonlinux_setup.md b/docs/install/amazonlinux_setup.md index 42a4fcb0eb89..a432d4815662 100644 --- a/docs/install/amazonlinux_setup.md +++ b/docs/install/amazonlinux_setup.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + diff --git a/docs/install/build_from_source.md b/docs/install/build_from_source.md index e807fb44b599..7b00b03abefe 100644 --- a/docs/install/build_from_source.md +++ b/docs/install/build_from_source.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Build MXNet from Source This document explains how to build MXNet from source code. @@ -28,11 +45,11 @@ MXNet's newest and most popular API is Gluon. Gluon is built into the Python bin - [Python (includes Gluon)](../api/python/index.html) - [C++](../api/c++/index.html) - [Clojure](../api/clojure/index.html) + - [Java](../api/java/index.html) - [Julia](../api/julia/index.html) - [Perl](../api/perl/index.html) - [R](../api/r/index.html) - [Scala](../api/scala/index.html) - - [Java](../api/java/index.html)
    diff --git a/docs/install/c_plus_plus.md b/docs/install/c_plus_plus.md index 6ad67e2803db..ee21014bc5f1 100644 --- a/docs/install/c_plus_plus.md +++ b/docs/install/c_plus_plus.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## Build the C++ package The C++ package has the same prerequisites as the MXNet library. diff --git a/docs/install/centos_setup.md b/docs/install/centos_setup.md index e5efe42a61dd..e01b68221bd9 100644 --- a/docs/install/centos_setup.md +++ b/docs/install/centos_setup.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Installing MXNet on CentOS and other non-Ubuntu Linux systems Step 1. Install build tools and git on `CentOS >= 7` and `Fedora >= 19`: diff --git a/docs/install/download.md b/docs/install/download.md index d7b9440e0a98..cf95c2344f14 100644 --- a/docs/install/download.md +++ b/docs/install/download.md @@ -1,9 +1,27 @@ + + + + + + + + + + + + + + + + + # Source Download These source archives are generated from tagged releases. Updates and patches will not have been applied. For any updates refer to the corresponding branches in the [GitHub repository](https://github.com/apache/incubator-mxnet). Choose your flavor of download from the following links: | Version | Source | PGP | SHA | |---------|-------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| 1.4.0 | [Download](https://www.apache.org/dyn/closer.cgi/incubator/mxnet/1.4.0/apache-mxnet-src-1.4.0-incubating.tar.gz) | [Download](https://apache.org/dist/incubator/mxnet/1.4.0/apache-mxnet-src-1.4.0-incubating.tar.gz.asc) | [Download](https://apache.org/dist/incubator/mxnet/1.4.0/apache-mxnet-src-1.4.0-incubating.tar.gz.sha512) | | 1.3.1 | [Download](https://www.apache.org/dyn/closer.cgi/incubator/mxnet/1.3.1/apache-mxnet-src-1.3.1-incubating.tar.gz) | [Download](https://apache.org/dist/incubator/mxnet/1.3.1/apache-mxnet-src-1.3.1-incubating.tar.gz.asc) | [Download](https://apache.org/dist/incubator/mxnet/1.3.1/apache-mxnet-src-1.3.1-incubating.tar.gz.sha512) | | 1.3.0 | [Download](https://archive.apache.org/dist/incubator/mxnet/1.3.0/apache-mxnet-src-1.3.0-incubating.tar.gz) | [Download](https://archive.apache.org/dist/incubator/mxnet/1.3.0/apache-mxnet-src-1.3.0-incubating.tar.gz.asc) | [Download](https://archive.apache.org/dist/incubator/mxnet/1.3.0/apache-mxnet-src-1.3.0-incubating.tar.gz.sha512) | | 1.2.1 | [Download](https://archive.apache.org/dist/incubator/mxnet/1.2.1/apache-mxnet-src-1.2.1-incubating.tar.gz) | [Download](https://archive.apache.org/dist/incubator/mxnet/1.2.1/apache-mxnet-src-1.2.1-incubating.tar.gz.asc) | [Download](https://archive.apache.org/dist/incubator/mxnet/1.2.1/apache-mxnet-src-1.2.1-incubating.tar.gz.sha512) | diff --git a/docs/install/index.md b/docs/install/index.md index ad3d083a7d02..eab1c82d7a88 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Installing MXNet ```eval_rst @@ -22,10 +39,11 @@ Indicate your preferred configuration. Then, follow the customized commands to install MXNet.
    -
      -
    • v1.3.1
      • +
    • v1.4.0
      • +
    • v1.3.1
    • v1.2.1
    • v1.1.0
    • v1.0.0
      • @@ -100,12 +118,19 @@ Indicate your preferred configuration. Then, follow the customized commands to i
      -
      +
      ``` $ pip install mxnet ``` +
      +
      + +``` +$ pip install mxnet==1.3.1 +``` +
      @@ -165,7 +190,7 @@ $ pip install mxnet --pre MXNet offers MKL pip packages that will be much faster when running on Intel hardware. Check the chart below for other options, refer to PyPI for other MXNet pip packages, or validate your MXNet installation. -pip packages +pip packages **NOTES:** @@ -234,12 +259,19 @@ To build from source, refer to the MXNet Ubuntu inst
      -
      +
      ``` $ pip install mxnet-cu92 ``` +
      +
      + +``` +$ pip install mxnet==1.3.1 +``` +
      @@ -292,7 +324,7 @@ $ pip install mxnet-cu92 --pre MXNet offers MKL pip packages that will be much faster when running on Intel hardware. Check the chart below for other options, refer to PyPI for other MXNet pip packages, or validate your MXNet installation. -pip packages +pip packages **NOTES:** @@ -541,11 +573,17 @@ Refer to the Perl
      -
      +
      ``` $ pip install mxnet ``` +
      +
      + +``` +$ pip install mxnet==1.3.1 +```
      @@ -602,7 +640,7 @@ $ pip install mxnet --pre MXNet offers MKL pip packages that will be much faster when running on Intel hardware. Check the chart below for other options, refer to PyPI for other MXNet pip packages, or validate your MXNet installation. -pip packages +pip packages **NOTES:** @@ -825,12 +863,19 @@ For more installation options, refer to the MXNet macOS
      -
      +
      ``` $ pip install mxnet ``` +
      +
      + +``` +$ pip install mxnet==1.3.1 +``` +
      @@ -883,7 +928,7 @@ $ pip install mxnet --pre MXNet offers MKL pip packages that will be much faster when running on Intel hardware. Check the chart below for other options, refer to PyPI for other MXNet pip packages, or validate your MXNet installation. -pip packages +pip packages **NOTES:** @@ -949,12 +994,19 @@ Refer to the MXNet Windows installation guide
      -
      +
      ``` $ pip install mxnet-cu92 ``` +
      +
      + +``` +$ pip install mxnet==1.3.1 +``` +
      @@ -1007,7 +1059,7 @@ $ pip install mxnet-cu92 --pre MXNet offers MKL pip packages that will be much faster when running on Intel hardware. Check the chart below for other options, refer to PyPI for other MXNet pip packages, or validate your MXNet installation. -pip packages +pip packages **NOTES:** @@ -1171,7 +1223,7 @@ MXNet should work on any cloud provider's CPU-only instances. Follow the Python
      -MXNet supports the Debian based Raspbian ARM based operating system so you can run MXNet on Raspberry Pi Devices. +MXNet supports the Debian based Raspbian ARM based operating system so you can run MXNet on Raspberry Pi 3B devices. These instructions will walk through how to build MXNet for the Raspberry Pi and install the Python bindings for the library. @@ -1179,6 +1231,9 @@ You can do a dockerized cross compilation build on your local machine or a nativ The complete MXNet library and its requirements can take almost 200MB of RAM, and loading large models with the library can take over 1GB of RAM. Because of this, we recommend running MXNet on the Raspberry Pi 3 or an equivalent device that has more than 1 GB of RAM and a Secure Digital (SD) card that has at least 4 GB of free memory. +## Quick installation +You can use this [pre-built Python wheel](wget https://mxnet-public.s3.amazonaws.com/install/raspbian/mxnet-1.5.0-py2.py3-none-any.whl) on a Raspberry Pi 3B with Stretch. You will likely need to install several dependencies to get MXNet to work. Refer to the following **Build** section for details. + **Cross compilation build (Experimental)** ## Docker installation @@ -1205,11 +1260,48 @@ ci/build.py -p armv7 ## Install -Create a virtualenv and install the package we created previously. +Your Pi will need several dependencies. + +Install MXNet dependencies with the following: +``` +sudo apt-get update +sudo apt-get install -y \ + apt-transport-https \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + git \ + libatlas-base-dev \ + libcurl4-openssl-dev \ + libjemalloc-dev \ + liblapack-dev \ + libopenblas-dev \ + libopencv-dev \ + libzmq3-dev \ + ninja-build \ + python-dev \ + software-properties-common \ + sudo \ + unzip \ + virtualenv \ + wget +``` +Install virtualenv with: +``` +sudo pip install virtualenv +``` +Create a Python 2.7 environment for MXNet with: +``` +virtualenv -p `which python` mxnet_py27 +``` +You may use Python 3, however the [wine bottle detection example](https://mxnet.incubator.apache.org/versions/master/tutorials/embedded/wine_detector.html) for the Pi with camera requires Python 2.7. + +Create a virtualenv and install the wheel we created previously, or the wheel that you downloaded. ``` -virtualenv -p `which python3` mxnet_py3 -source mxnet_py3/bin/activate +virtualenv -p `which python3` mxnet_py27 +source mxnet_py27/bin/activate pip install mxnet-x.x.x-py2.py3-none-any.whl ``` @@ -1240,7 +1332,7 @@ Install these dependencies using the following commands in any directory: ``` sudo apt-get update - sudo apt-get -y install git cmake ninja-build build-essential g++-4.9 c++-4.9 liblapack* libblas* libopencv* libopenblas* python3-dev virtualenv + sudo apt-get -y install git cmake ninja-build build-essential g++-4.9 c++-4.9 liblapack* libblas* libopencv* libopenblas* python3-dev python-dev virtualenv ``` Clone the MXNet source code repository using the following `git` command in your home directory: diff --git a/docs/install/java_setup.md b/docs/install/java_setup.md index bd20c9596013..569f439f8b02 100644 --- a/docs/install/java_setup.md +++ b/docs/install/java_setup.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Setup the MXNet Package for Java The following instructions are provided for macOS and Ubuntu. Windows is not yet available. diff --git a/docs/install/osx_setup.md b/docs/install/osx_setup.md index 7d90d3d456f6..84e58282d766 100644 --- a/docs/install/osx_setup.md +++ b/docs/install/osx_setup.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Installing MXNet from source on OS X (Mac) **NOTE:** For prebuild MXNet with Python installation, please refer to the [new install guide](http://mxnet.io/install/index.html). diff --git a/docs/install/raspbian_setup.md b/docs/install/raspbian_setup.md index 42a4fcb0eb89..896d4721370b 100644 --- a/docs/install/raspbian_setup.md +++ b/docs/install/raspbian_setup.md @@ -1,8 +1,25 @@ + + + + + + + + + + + + + + + + + - +

      - + This content is moved to a new MXNet install page. Redirecting...

      diff --git a/docs/install/scala_setup.md b/docs/install/scala_setup.md index 9ee9ceac3a3c..15a2def1ef38 100644 --- a/docs/install/scala_setup.md +++ b/docs/install/scala_setup.md @@ -1,8 +1,26 @@ + + + + + + + + + + + + + + + + + # Setup the MXNet Package for Scala The following instructions are provided for macOS and Ubuntu. Windows is not yet available. **Note:** If you use IntelliJ or a similar IDE, you may want to follow the [MXNet-Scala on IntelliJ tutorial](../tutorials/scala/mxnet_scala_on_intellij.html) instead of these instructions. +**Note:** Currently, we only support scala 2.11
      @@ -97,6 +115,33 @@ mvn install
      +## Interpreter + +To run the scala interpreter, first download and install scala 2.11.x (run `scala -version` to make sure you have the right version installed.** + +### Installing the Interpreter + +**Ubuntu*** + +``` +sudo apt-get install scala +``` + +**macOS*** + +``` +brew install scala@2.11 +``` + +Then, add scala to your path by following the instructions output by homebrew. + +### Running the Interpreter + +To run the interpreter, download the appropriate mxnet jar from [the maven repository](https://search.maven.org/search?q=g:org.apache.mxnet) or build from source following the instructions above. + +Then, run `scala -cp {path/to/mxnet-full_2.11-os-version.jar}` to start it. +If you receive a "NumberFormatException" when running the interpreter, run `export TERM=xterm-color` before starting the interpreter. + ## Documentation Scaladocs are generated as part of the docs build pipeline. You can find them published in the [Scala API](http://mxnet.incubator.apache.org/api/scala/index.html) section of the website or by going to the [scaladocs output](https://mxnet.incubator.apache.org/api/scala/docs/index.html#org.apache.mxnet.package) directly. diff --git a/docs/install/tx2_setup.md b/docs/install/tx2_setup.md index 42a4fcb0eb89..a432d4815662 100644 --- a/docs/install/tx2_setup.md +++ b/docs/install/tx2_setup.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + diff --git a/docs/install/ubuntu_setup.md b/docs/install/ubuntu_setup.md index bf964182b50a..24f9ab98e6e4 100644 --- a/docs/install/ubuntu_setup.md +++ b/docs/install/ubuntu_setup.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Installing MXNet on Ubuntu The following installation instructions are for installing MXNet on computers running **Ubuntu 16.04**. Support for later versions of Ubuntu is [not yet available](#contributions). @@ -80,7 +97,7 @@ Alternatively, you can use the table below to select the package that suits your The following table presents the pip packages that are recommended for each version of MXNet. -![pip package table](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/install/pip-packages.png) +![pip package table](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/install/pip-packages-1.4.0.png) To install an older version of MXNet with one of the packages in the previous table add `==` with the version you require. For example for version 1.1.0 of MXNet with CUDA 8, you would use `pip install mxnet-cu80==1.1.0`. diff --git a/docs/install/validate_mxnet.md b/docs/install/validate_mxnet.md index dfe8d063f602..c31463b63eee 100644 --- a/docs/install/validate_mxnet.md +++ b/docs/install/validate_mxnet.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Validate Your MXNet Installation - [Python](#python) diff --git a/docs/install/windows_setup.md b/docs/install/windows_setup.md old mode 100755 new mode 100644 index b34936140aea..3c3da5349235 --- a/docs/install/windows_setup.md +++ b/docs/install/windows_setup.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Installing MXNet on Windows The following describes how to install with pip for computers with CPUs, Intel CPUs, and NVIDIA GPUs. Further along in the document you can learn how to build MXNet from source on Windows, or how to install packages that support different language APIs to MXNet. diff --git a/docs/model_zoo/index.md b/docs/model_zoo/index.md index 034d360b985a..cfe13caa0bf0 100644 --- a/docs/model_zoo/index.md +++ b/docs/model_zoo/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Model Zoo MXNet features fast implementations of many state-of-the-art models reported in the academic literature. This Model Zoo is an diff --git a/docs/settings.ini b/docs/settings.ini index e16177604e8d..ec64dd784cdb 100644 --- a/docs/settings.ini +++ b/docs/settings.ini @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [mxnet] build_mxnet = 0 diff --git a/docs/tutorials/basic/data.md b/docs/tutorials/basic/data.md index 4a682e83f9fe..00c87d0c37c5 100644 --- a/docs/tutorials/basic/data.md +++ b/docs/tutorials/basic/data.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Iterators - Loading data In this tutorial, we focus on how to feed data into a training or inference program. Most training and inference modules in MXNet accept data iterators, diff --git a/docs/tutorials/basic/index.md b/docs/tutorials/basic/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/basic/index.md +++ b/docs/tutorials/basic/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/basic/module.md b/docs/tutorials/basic/module.md index f7a4d6e25de7..64a1e9482ea8 100644 --- a/docs/tutorials/basic/module.md +++ b/docs/tutorials/basic/module.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Module - Neural network training and inference diff --git a/docs/tutorials/basic/ndarray.md b/docs/tutorials/basic/ndarray.md index 2c171f2627e8..be5dbcf29e88 100644 --- a/docs/tutorials/basic/ndarray.md +++ b/docs/tutorials/basic/ndarray.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # NDArray - Imperative tensor operations on CPU/GPU In _MXNet_, `NDArray` is the core data structure for all mathematical diff --git a/docs/tutorials/basic/ndarray_indexing.md b/docs/tutorials/basic/ndarray_indexing.md index 35dd8c17f675..00a231351d7d 100644 --- a/docs/tutorials/basic/ndarray_indexing.md +++ b/docs/tutorials/basic/ndarray_indexing.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # NDArray Indexing - Array indexing features diff --git a/docs/tutorials/basic/reshape_transpose.md b/docs/tutorials/basic/reshape_transpose.md index 999b22ca2f7e..40bdb38f3ec2 100644 --- a/docs/tutorials/basic/reshape_transpose.md +++ b/docs/tutorials/basic/reshape_transpose.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## Difference between reshape and transpose operators What does it mean if MXNet gives you an error like the this? diff --git a/docs/tutorials/basic/symbol.md b/docs/tutorials/basic/symbol.md index 5e1e3cd8c62f..cff12aca3c95 100644 --- a/docs/tutorials/basic/symbol.md +++ b/docs/tutorials/basic/symbol.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Symbol - Neural network graphs In a [previous tutorial](http://mxnet.io/tutorials/basic/ndarray.html), we introduced `NDArray`, diff --git a/docs/tutorials/c++/basics.md b/docs/tutorials/c++/basics.md index aa73a7363b1c..ddc5595cf3c0 100644 --- a/docs/tutorials/c++/basics.md +++ b/docs/tutorials/c++/basics.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Basics ====== diff --git a/docs/tutorials/c++/index.md b/docs/tutorials/c++/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/c++/index.md +++ b/docs/tutorials/c++/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md b/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md index ab55a0e787f1..ca4a75ef3b54 100644 --- a/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md +++ b/docs/tutorials/c++/mxnet_cpp_inference_tutorial.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet C++ API inference tutorial ## Overview diff --git a/docs/tutorials/c++/subgraphAPI.md b/docs/tutorials/c++/subgraphAPI.md index 0ae4341b287c..b834df8741b5 100644 --- a/docs/tutorials/c++/subgraphAPI.md +++ b/docs/tutorials/c++/subgraphAPI.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## Subgraph API The subgraph API has been proposed and implemented as the default mechanism for integrating backend libraries to MXNet. The subgraph API is a very flexible interface. Although it was proposed as an integration mechanism, it has been used as a tool for manipulating NNVM graphs for graph-level optimizations, such as operator fusion. diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md index 4b6a23136b5d..88aa383cb011 100644 --- a/docs/tutorials/control_flow/ControlFlowTutorial.md +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Hybridize Gluon models with control flows. MXNet currently provides three control flow operators: `cond`, `foreach` and `while_loop`. Like other MXNet operators, they all have a version for NDArray and a version for Symbol. These two versions have exactly the same semantics. We can take advantage of this and use them in Gluon to hybridize models. diff --git a/docs/tutorials/control_flow/index.md b/docs/tutorials/control_flow/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/control_flow/index.md +++ b/docs/tutorials/control_flow/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/embedded/index.md b/docs/tutorials/embedded/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/embedded/index.md +++ b/docs/tutorials/embedded/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/embedded/wine_detector.md b/docs/tutorials/embedded/wine_detector.md index 65e6fbaa4d91..f0ae8273203e 100644 --- a/docs/tutorials/embedded/wine_detector.md +++ b/docs/tutorials/embedded/wine_detector.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Real-time Object Detection with MXNet On The Raspberry Pi This tutorial shows developers who work with the Raspberry Pi or similar embedded ARM-based devices how to compile MXNet for those devices and run a pretrained deep network model. It also shows how to use AWS IoT to manage and monitor MXNet models running on your devices. diff --git a/docs/tutorials/gluon/autograd.md b/docs/tutorials/gluon/autograd.md index 4b296dd2dd5b..4d8a042c9b03 100644 --- a/docs/tutorials/gluon/autograd.md +++ b/docs/tutorials/gluon/autograd.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Automatic differentiation MXNet supports automatic differentiation with the `autograd` package. diff --git a/docs/tutorials/gluon/custom_layer.md b/docs/tutorials/gluon/custom_layer.md index 97bdf05aff58..40ba0823a9d5 100644 --- a/docs/tutorials/gluon/custom_layer.md +++ b/docs/tutorials/gluon/custom_layer.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # How to write a custom layer in Apache MxNet Gluon API diff --git a/docs/tutorials/gluon/customop.md b/docs/tutorials/gluon/customop.md index df10788d6788..eae0344c8702 100644 --- a/docs/tutorials/gluon/customop.md +++ b/docs/tutorials/gluon/customop.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Creating custom operators with numpy diff --git a/docs/tutorials/gluon/data_augmentation.md b/docs/tutorials/gluon/data_augmentation.md index df7462c33044..356d335a8ca2 100644 --- a/docs/tutorials/gluon/data_augmentation.md +++ b/docs/tutorials/gluon/data_augmentation.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Methods of applying data augmentation (Gluon API) Data Augmentation is a regularization technique that's used to avoid overfitting when training Machine Learning models. Although the technique can be applied in a variety of domains, it's very common in Computer Vision. Adjustments are made to the original images in the training dataset before being used in training. Some example adjustments include translating, cropping, scaling, rotating, changing brightness and contrast. We do this to reduce the dependence of the model on spurious characteristics; e.g. training data may only contain faces that fill 1/4 of the image, so the model trained without data augmentation might unhelpfully learn that faces can only be of this size. diff --git a/docs/tutorials/gluon/datasets.md b/docs/tutorials/gluon/datasets.md index 0b0038def633..c029124af8b6 100644 --- a/docs/tutorials/gluon/datasets.md +++ b/docs/tutorials/gluon/datasets.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon `Dataset`s and `DataLoader` diff --git a/docs/tutorials/gluon/gluon.md b/docs/tutorials/gluon/gluon.md index 518e99905c04..d9a6e93af689 100644 --- a/docs/tutorials/gluon/gluon.md +++ b/docs/tutorials/gluon/gluon.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon - Neural network building blocks Gluon package is a high-level interface for MXNet designed to be easy to use while diff --git a/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md index 36c7a2e4dafa..42a65b352199 100644 --- a/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md +++ b/docs/tutorials/gluon/gluon_from_experiment_to_deployment.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gluon: from experiment to deployment, an end to end tutorial @@ -52,7 +69,6 @@ Now let's first import necessary packages: import math import os import time -from multiprocessing import cpu_count from mxnet import autograd from mxnet import gluon, init @@ -77,7 +93,8 @@ lr_factor = 0.75 lr_epochs = [10, 20, 30] num_gpus = mx.context.num_gpus() -num_workers = cpu_count() +# you can replace num_workers with the number of cores on you device +num_workers = 8 ctx = [mx.gpu(i) for i in range(num_gpus)] if num_gpus > 0 else [mx.cpu()] batch_size = per_device_batch_size * max(num_gpus, 1) ``` diff --git a/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md b/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md index c82c63edbc2b..1c285a41d76b 100644 --- a/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md +++ b/docs/tutorials/gluon/gotchas_numpy_in_mxnet.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Gotchas using NumPy in Apache MXNet diff --git a/docs/tutorials/gluon/hybrid.md b/docs/tutorials/gluon/hybrid.md index 17e9e1b20d74..b7725386336e 100644 --- a/docs/tutorials/gluon/hybrid.md +++ b/docs/tutorials/gluon/hybrid.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Hybrid - Faster training and easy deployment *Related Content:* diff --git a/docs/tutorials/gluon/index.md b/docs/tutorials/gluon/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/gluon/index.md +++ b/docs/tutorials/gluon/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/gluon/info_gan.md b/docs/tutorials/gluon/info_gan.md index 8b2668ab22e6..93fd6cb5e7fd 100644 --- a/docs/tutorials/gluon/info_gan.md +++ b/docs/tutorials/gluon/info_gan.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Image similarity search with InfoGAN diff --git a/docs/tutorials/gluon/learning_rate_finder.md b/docs/tutorials/gluon/learning_rate_finder.md index b571a53f674c..30c66e302766 100644 --- a/docs/tutorials/gluon/learning_rate_finder.md +++ b/docs/tutorials/gluon/learning_rate_finder.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Learning Rate Finder diff --git a/docs/tutorials/gluon/learning_rate_schedules.md b/docs/tutorials/gluon/learning_rate_schedules.md index 88b109e7f33e..46c79ebc249b 100644 --- a/docs/tutorials/gluon/learning_rate_schedules.md +++ b/docs/tutorials/gluon/learning_rate_schedules.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Learning Rate Schedules diff --git a/docs/tutorials/gluon/learning_rate_schedules_advanced.md b/docs/tutorials/gluon/learning_rate_schedules_advanced.md index bdaf0a9ba38d..287f70d535e8 100644 --- a/docs/tutorials/gluon/learning_rate_schedules_advanced.md +++ b/docs/tutorials/gluon/learning_rate_schedules_advanced.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Advanced Learning Rate Schedules diff --git a/docs/tutorials/gluon/logistic_regression_explained.md b/docs/tutorials/gluon/logistic_regression_explained.md index 577a91413b33..57ed56eb8f4f 100644 --- a/docs/tutorials/gluon/logistic_regression_explained.md +++ b/docs/tutorials/gluon/logistic_regression_explained.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Logistic regression using Gluon API explained diff --git a/docs/tutorials/gluon/mnist.md b/docs/tutorials/gluon/mnist.md index 35fb40521f62..828bfd51f601 100644 --- a/docs/tutorials/gluon/mnist.md +++ b/docs/tutorials/gluon/mnist.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Hand-written Digit Recognition In this tutorial, we'll give you a step-by-step walkthrough of building a hand-written digit classifier using the [MNIST](https://en.wikipedia.org/wiki/MNIST_database) dataset. diff --git a/docs/tutorials/gluon/naming.md b/docs/tutorials/gluon/naming.md index 3606a03dcbd2..e667ad3cb79e 100644 --- a/docs/tutorials/gluon/naming.md +++ b/docs/tutorials/gluon/naming.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Naming of Gluon Parameter and Blocks diff --git a/docs/tutorials/gluon/ndarray.md b/docs/tutorials/gluon/ndarray.md index 7cf08a88cbf3..f1503aec18db 100644 --- a/docs/tutorials/gluon/ndarray.md +++ b/docs/tutorials/gluon/ndarray.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # NDArray - Scientific computing on CPU and GPU NDArray is a tensor data structure similar to numpy's multi-dimensional array. diff --git a/docs/tutorials/gluon/pretrained_models.md b/docs/tutorials/gluon/pretrained_models.md index 0de5fdd0b44f..27ec1a95b8cd 100644 --- a/docs/tutorials/gluon/pretrained_models.md +++ b/docs/tutorials/gluon/pretrained_models.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Using pre-trained models in MXNet diff --git a/docs/tutorials/gluon/save_load_params.md b/docs/tutorials/gluon/save_load_params.md index ebc8103e7b45..ffebefdf80e1 100644 --- a/docs/tutorials/gluon/save_load_params.md +++ b/docs/tutorials/gluon/save_load_params.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Saving and Loading Gluon Models Training large models take a lot of time and it is a good idea to save the trained models to files to avoid training them again and again. There are a number of reasons to do this. For example, you might want to do inference on a machine that is different from the one where the model was trained. Sometimes model's performance on validation set decreases towards the end of the training because of overfitting. If you saved your model parameters after every epoch, at the end you can decide to use the model that performs best on the validation set. Another reason would be to train your model using one language (like Python that has a lot of tools for training) and run inference using a different language (like Scala probably because your application is built on Scala). diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index cad9099fcc71..7e0ffaa3f72a 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/java/index.md b/docs/tutorials/java/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/java/index.md +++ b/docs/tutorials/java/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/java/mxnet_java_on_intellij.md b/docs/tutorials/java/mxnet_java_on_intellij.md index ef2c009f66e8..ce84788c846e 100644 --- a/docs/tutorials/java/mxnet_java_on_intellij.md +++ b/docs/tutorials/java/mxnet_java_on_intellij.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Run MXNet Java Examples Using the IntelliJ IDE (macOS) This tutorial guides you through setting up a simple Java project in IntelliJ IDE on macOS and demonstrates usage of the MXNet Java APIs. diff --git a/docs/tutorials/java/ssd_inference.md b/docs/tutorials/java/ssd_inference.md index 3a20329f9a91..1117bbdcfa5b 100644 --- a/docs/tutorials/java/ssd_inference.md +++ b/docs/tutorials/java/ssd_inference.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Multi Object Detection using pre-trained SSD Model via Java Inference APIs This tutorial shows how to use MXNet Java Inference APIs to run inference on a pre-trained Single Shot Detector (SSD) Model. diff --git a/docs/tutorials/nlp/cnn.md b/docs/tutorials/nlp/cnn.md index b3d7d0d38941..e671de3a1f57 100644 --- a/docs/tutorials/nlp/cnn.md +++ b/docs/tutorials/nlp/cnn.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Text Classification Using a Convolutional Neural Network on MXNet This tutorial is based of Yoon Kim's [paper](https://arxiv.org/abs/1408.5882) on using convolutional neural networks for sentence sentiment classification. The tutorial has been tested on MXNet 1.0 running under Python 2.7 and Python 3.6. diff --git a/docs/tutorials/nlp/index.md b/docs/tutorials/nlp/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/nlp/index.md +++ b/docs/tutorials/nlp/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/onnx/export_mxnet_to_onnx.md b/docs/tutorials/onnx/export_mxnet_to_onnx.md index 3f925c7b5b84..c0f8d92901f6 100644 --- a/docs/tutorials/onnx/export_mxnet_to_onnx.md +++ b/docs/tutorials/onnx/export_mxnet_to_onnx.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Exporting MXNet model to ONNX format diff --git a/docs/tutorials/onnx/fine_tuning_gluon.md b/docs/tutorials/onnx/fine_tuning_gluon.md index 750a6757272f..dd0c0e93e862 100644 --- a/docs/tutorials/onnx/fine_tuning_gluon.md +++ b/docs/tutorials/onnx/fine_tuning_gluon.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Fine-tuning an ONNX model with MXNet/Gluon diff --git a/docs/tutorials/onnx/index.md b/docs/tutorials/onnx/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/onnx/index.md +++ b/docs/tutorials/onnx/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/onnx/inference_on_onnx_model.md b/docs/tutorials/onnx/inference_on_onnx_model.md index b2522ad0c1f1..f12e050fcc73 100644 --- a/docs/tutorials/onnx/inference_on_onnx_model.md +++ b/docs/tutorials/onnx/inference_on_onnx_model.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Running inference on MXNet/Gluon from an ONNX model diff --git a/docs/tutorials/onnx/super_resolution.md b/docs/tutorials/onnx/super_resolution.md index 36c06b743c8e..db5932949f14 100644 --- a/docs/tutorials/onnx/super_resolution.md +++ b/docs/tutorials/onnx/super_resolution.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Importing an ONNX model into MXNet In this tutorial we will: diff --git a/docs/tutorials/python/data_augmentation.md b/docs/tutorials/python/data_augmentation.md index e4dbbb672997..45a4e3d8c86d 100644 --- a/docs/tutorials/python/data_augmentation.md +++ b/docs/tutorials/python/data_augmentation.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Methods of applying data augmentation (Module API) Data Augmentation is a regularization technique that's used to avoid overfitting when training Machine Learning models. Although the technique can be applied in a variety of domains, it's very common in Computer Vision. Adjustments are made to the original images in the training dataset before being used in training. Some example adjustments include translating, cropping, scaling, rotating, changing brightness and contrast. We do this to reduce the dependence of the model on spurious characteristics; e.g. training data may only contain faces that fill 1/4 of the image, so the model trained without data augmentation might unhelpfully learn that faces can only be of this size. diff --git a/docs/tutorials/python/data_augmentation_with_masks.md b/docs/tutorials/python/data_augmentation_with_masks.md index ac587ac2f5e2..66beece720a9 100644 --- a/docs/tutorials/python/data_augmentation_with_masks.md +++ b/docs/tutorials/python/data_augmentation_with_masks.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Data Augmentation with Masks diff --git a/docs/tutorials/python/index.md b/docs/tutorials/python/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/python/index.md +++ b/docs/tutorials/python/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/python/kvstore.md b/docs/tutorials/python/kvstore.md index 3e6bbf12c393..42debab9b83e 100644 --- a/docs/tutorials/python/kvstore.md +++ b/docs/tutorials/python/kvstore.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Distributed Key-Value Store KVStore is a place for data sharing. Think of it as a single object shared diff --git a/docs/tutorials/python/linear-regression.md b/docs/tutorials/python/linear-regression.md index fd336ad2aed5..47d1acae9a14 100644 --- a/docs/tutorials/python/linear-regression.md +++ b/docs/tutorials/python/linear-regression.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Linear Regression In this tutorial we'll walk through how one can implement *linear regression* using MXNet APIs. diff --git a/docs/tutorials/python/matrix_factorization.md b/docs/tutorials/python/matrix_factorization.md index 154fa4b3e127..cfe73a4856e0 100644 --- a/docs/tutorials/python/matrix_factorization.md +++ b/docs/tutorials/python/matrix_factorization.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Matrix Factorization In a recommendation system, there is a group of users and a set of items. Given diff --git a/docs/tutorials/python/mnist.md b/docs/tutorials/python/mnist.md index df949d487b63..9d641b36c202 100644 --- a/docs/tutorials/python/mnist.md +++ b/docs/tutorials/python/mnist.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Handwritten Digit Recognition In this tutorial, we'll give you a step by step walk-through of how to build a hand-written digit classifier using the [MNIST](https://en.wikipedia.org/wiki/MNIST_database) dataset. For someone new to deep learning, this exercise is arguably the "Hello World" equivalent. diff --git a/docs/tutorials/python/predict_image.md b/docs/tutorials/python/predict_image.md index 8be98d991366..0721abd6a731 100644 --- a/docs/tutorials/python/predict_image.md +++ b/docs/tutorials/python/predict_image.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Predict with pre-trained models This tutorial explains how to recognize objects in an image with a pre-trained model, and how to perform feature extraction. diff --git a/docs/tutorials/python/profiler.md b/docs/tutorials/python/profiler.md index 7dcda10f11b8..fe7611aa538f 100644 --- a/docs/tutorials/python/profiler.md +++ b/docs/tutorials/python/profiler.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Profiling MXNet Models It is often helpful to understand what operations take how much time while running a model. This helps optimize the model to run faster. In this tutorial, we will learn how to profile MXNet models to measure their running time and memory consumption using the MXNet profiler. diff --git a/docs/tutorials/python/types_of_data_augmentation.md b/docs/tutorials/python/types_of_data_augmentation.md index 4308932bf483..63ad468f85f6 100644 --- a/docs/tutorials/python/types_of_data_augmentation.md +++ b/docs/tutorials/python/types_of_data_augmentation.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Types of Data Augmentation diff --git a/docs/tutorials/r/CallbackFunction.md b/docs/tutorials/r/CallbackFunction.md index 103352dd2907..1258c527e456 100644 --- a/docs/tutorials/r/CallbackFunction.md +++ b/docs/tutorials/r/CallbackFunction.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Callback Function ====================================== diff --git a/docs/tutorials/r/CustomIterator.md b/docs/tutorials/r/CustomIterator.md index 1ad634bcd669..377838d93e46 100644 --- a/docs/tutorials/r/CustomIterator.md +++ b/docs/tutorials/r/CustomIterator.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Custom Iterator Tutorial ====================================== diff --git a/docs/tutorials/r/CustomLossFunction.md b/docs/tutorials/r/CustomLossFunction.md index afb99518894c..6b60886bceba 100644 --- a/docs/tutorials/r/CustomLossFunction.md +++ b/docs/tutorials/r/CustomLossFunction.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Customized loss function ====================================== diff --git a/docs/tutorials/r/MultidimLstm.md b/docs/tutorials/r/MultidimLstm.md index 8692086d180b..5abd38d2e02c 100644 --- a/docs/tutorials/r/MultidimLstm.md +++ b/docs/tutorials/r/MultidimLstm.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + LSTM time series example ============================================= diff --git a/docs/tutorials/r/charRnnModel.md b/docs/tutorials/r/charRnnModel.md index cb21e77559b5..ed873d49214d 100644 --- a/docs/tutorials/r/charRnnModel.md +++ b/docs/tutorials/r/charRnnModel.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Character-level Language Model using RNN diff --git a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md index b2f2035426ec..1344285dac59 100644 --- a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md +++ b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Classify Images with a PreTrained Model ================================================= MXNet is a flexible and efficient deep learning framework. One of the interesting things that a deep learning diff --git a/docs/tutorials/r/fiveMinutesNeuralNetwork.md b/docs/tutorials/r/fiveMinutesNeuralNetwork.md index 6d79cd288d2c..39775b3ab640 100644 --- a/docs/tutorials/r/fiveMinutesNeuralNetwork.md +++ b/docs/tutorials/r/fiveMinutesNeuralNetwork.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Develop a Neural Network with MXNet in Five Minutes ============================================= diff --git a/docs/tutorials/r/index.md b/docs/tutorials/r/index.md index fbc8911f2a6d..13db9694864d 100644 --- a/docs/tutorials/r/index.md +++ b/docs/tutorials/r/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # R Tutorials These tutorials introduce a few fundamental concepts in deep learning and how to implement them in R using _MXNet_. diff --git a/docs/tutorials/r/mnistCompetition.md b/docs/tutorials/r/mnistCompetition.md index ed3c2827011d..f59aabd77905 100644 --- a/docs/tutorials/r/mnistCompetition.md +++ b/docs/tutorials/r/mnistCompetition.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Handwritten Digits Classification Competition ============================================= diff --git a/docs/tutorials/r/ndarray.md b/docs/tutorials/r/ndarray.md index cb7639a8a44d..e935a5052cf3 100644 --- a/docs/tutorials/r/ndarray.md +++ b/docs/tutorials/r/ndarray.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # NDArray: Vectorized Tensor Computations on CPUs and GPUs `NDArray` is the basic vectorized operation unit in MXNet for matrix and tensor computations. diff --git a/docs/tutorials/r/symbol.md b/docs/tutorials/r/symbol.md index 4a87643b9f50..d60c5fbfbc59 100644 --- a/docs/tutorials/r/symbol.md +++ b/docs/tutorials/r/symbol.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Symbol and Automatic Differentiation The computational unit `NDArray` requires a way to construct neural networks. MXNet provides a symbolic interface, named Symbol, to do this. Symbol combines both flexibility and efficiency. diff --git a/docs/tutorials/scala/char_lstm.md b/docs/tutorials/scala/char_lstm.md index 4d6a5aee921e..aca08dc79920 100644 --- a/docs/tutorials/scala/char_lstm.md +++ b/docs/tutorials/scala/char_lstm.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Developing a Character-level Language model This tutorial shows how to train a character-level language model with a multilayer recurrent neural network (RNN) using Scala. This model takes one text file as input and trains an RNN that learns to predict the next character in the sequence. In this tutorial, you train a multilayer LSTM (Long Short-Term Memory) network that generates relevant text using Barack Obama's speech patterns. @@ -54,11 +71,7 @@ In this tutorial, you will accomplish the following: ## Prerequisites -To complete this tutorial, you need: - -- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/install/index.html) -- [Scala 2.11.8](https://www.scala-lang.org/download/2.11.8.html) -- [Maven 3](https://maven.apache.org/install.html) +To complete this tutorial, setup and run the scala interpreter by following the [instructions](https://mxnet.incubator.apache.org/install/scala_setup.html#interpreter). ## Download the Data diff --git a/docs/tutorials/scala/index.md b/docs/tutorials/scala/index.md index f14337f90f08..1e03345a1e45 100644 --- a/docs/tutorials/scala/index.md +++ b/docs/tutorials/scala/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet-Scala Tutorials ## Installation & Setup diff --git a/docs/tutorials/scala/mnist.md b/docs/tutorials/scala/mnist.md index 79f2129ef0ef..89bc203231f7 100644 --- a/docs/tutorials/scala/mnist.md +++ b/docs/tutorials/scala/mnist.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Handwritten Digit Recognition This Scala tutorial guides you through a classic computer vision application: identifying hand written digits. diff --git a/docs/tutorials/scala/mxnet_scala_on_intellij.md b/docs/tutorials/scala/mxnet_scala_on_intellij.md index 769a6b4fe506..f1d15cb3c5b5 100644 --- a/docs/tutorials/scala/mxnet_scala_on_intellij.md +++ b/docs/tutorials/scala/mxnet_scala_on_intellij.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Run MXNet Scala Examples Using the IntelliJ IDE (macOS) This tutorial guides you through setting up a Scala project in the IntelliJ IDE on macOS, and shows how to use the MXNet package from your application. diff --git a/docs/tutorials/sparse/csr.md b/docs/tutorials/sparse/csr.md index 0aede1ab4313..25de1f716045 100644 --- a/docs/tutorials/sparse/csr.md +++ b/docs/tutorials/sparse/csr.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # CSRNDArray - NDArray in Compressed Sparse Row Storage Format diff --git a/docs/tutorials/sparse/index.md b/docs/tutorials/sparse/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/sparse/index.md +++ b/docs/tutorials/sparse/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/sparse/row_sparse.md b/docs/tutorials/sparse/row_sparse.md index 46a5edad075e..f89de3d5b75e 100644 --- a/docs/tutorials/sparse/row_sparse.md +++ b/docs/tutorials/sparse/row_sparse.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # RowSparseNDArray - NDArray for Sparse Gradient Updates diff --git a/docs/tutorials/sparse/train.md b/docs/tutorials/sparse/train.md index fde4c0e65521..2f315cfa1010 100644 --- a/docs/tutorials/sparse/train.md +++ b/docs/tutorials/sparse/train.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Train a Linear Regression Model with Sparse Symbols In previous tutorials, we introduced `CSRNDArray` and `RowSparseNDArray`, diff --git a/docs/tutorials/speech_recognition/ctc.md b/docs/tutorials/speech_recognition/ctc.md index 0b01fb48999c..948e681de52f 100644 --- a/docs/tutorials/speech_recognition/ctc.md +++ b/docs/tutorials/speech_recognition/ctc.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Connectionist Temporal Classification ```python diff --git a/docs/tutorials/speech_recognition/index.md b/docs/tutorials/speech_recognition/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/speech_recognition/index.md +++ b/docs/tutorials/speech_recognition/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/tensorrt/index.md b/docs/tutorials/tensorrt/index.md index 9515a5b9fd1e..bc10874886ef 100644 --- a/docs/tutorials/tensorrt/index.md +++ b/docs/tutorials/tensorrt/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/tensorrt/inference_with_trt.md b/docs/tutorials/tensorrt/inference_with_trt.md index 489a92ec1014..ff3cddf3d574 100644 --- a/docs/tutorials/tensorrt/inference_with_trt.md +++ b/docs/tutorials/tensorrt/inference_with_trt.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Optimizing Deep Learning Computation Graphs with TensorRT NVIDIA's TensorRT is a deep learning library that has been shown to provide large speedups when used for network inference. MXNet 1.3.0 is shipping with experimental integrated support for TensorRT. This means MXNet users can noew make use of this acceleration library to efficiently run their networks. In this blog post we'll see how to install, enable and run TensorRT with MXNet. We'll also give some insight into what is happening behind the scenes in MXNet to enable TensorRT graph execution. diff --git a/docs/tutorials/unsupervised_learning/gan.md b/docs/tutorials/unsupervised_learning/gan.md index 0efdc5565519..ca0fb15e01c5 100644 --- a/docs/tutorials/unsupervised_learning/gan.md +++ b/docs/tutorials/unsupervised_learning/gan.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Generative Adversarial Network (GAN) diff --git a/docs/tutorials/unsupervised_learning/index.md b/docs/tutorials/unsupervised_learning/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/unsupervised_learning/index.md +++ b/docs/tutorials/unsupervised_learning/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/vision/cnn_visualization.md b/docs/tutorials/vision/cnn_visualization.md index 5ded6f1587e0..1eb89e104264 100644 --- a/docs/tutorials/vision/cnn_visualization.md +++ b/docs/tutorials/vision/cnn_visualization.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Visualizing Decisions of Convolutional Neural Networks Convolutional Neural Networks have made a lot of progress in Computer Vision. Their accuracy is as good as humans in some tasks. However, it remains difficult to explain the predictions of convolutional neural networks, as they lack the interpretability offered by other models such as decision trees. diff --git a/docs/tutorials/vision/index.md b/docs/tutorials/vision/index.md index 87d72894424f..faf6526fb824 100644 --- a/docs/tutorials/vision/index.md +++ b/docs/tutorials/vision/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tutorials ```eval_rst diff --git a/docs/tutorials/vision/large_scale_classification.md b/docs/tutorials/vision/large_scale_classification.md index aac03e4dd903..36a0ff6c9daa 100644 --- a/docs/tutorials/vision/large_scale_classification.md +++ b/docs/tutorials/vision/large_scale_classification.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Large Scale Image Classification Training a neural network with a large number of images presents several challenges. Even with the latest GPUs, it is not possible to train large networks using a large number of images in a reasonable amount of time using a single GPU. This problem can be somewhat mitigated by using multiple GPUs in a single machine. But there is a limit to the number of GPUs that can be attached to one machine (typically 8 or 16). This tutorial explains how to train large networks with terabytes of data using multiple machines each containing multiple GPUs. diff --git a/example/README.md b/example/README.md index dea7e289e6cd..18022beb7598 100644 --- a/example/README.md +++ b/example/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Examples This page contains a curated list of awesome MXNet examples, tutorials and blogs. It is inspired by [awesome-php](https://github.com/ziadoz/awesome-php) and [awesome-machine-learning](https://github.com/josephmisiti/awesome-machine-learning). See also [Awesome-MXNet](https://github.com/chinakook/Awesome-MXNet) for a similar list. @@ -111,6 +128,7 @@ If your tutorial depends on specific packages, simply add them to this provision * [Numpy Operator Customization](numpy-ops) - Examplea on quick customize new ops with Numpy * [Profiling](profiler) - generate profiling results in json files * [Python How To](python-howto) - a variety of Python examples +* [Quantization and Calibration Examples](quantization) - examples of quantizing a FP32 model to INT8 and performing low-precision inference with Intel MKL-DNN on CPU or cuDNN on GPU * [R-CNN](rcnn) - R-CNN with distributed implementation and data parallelization * [Recommender Systems](recommenders) - examples of how to build various kinds of recommender systems * [Reinforcement Learning](reinforcement-learning) - a variety of reinforcement learning examples diff --git a/example/adversary/README.md b/example/adversary/README.md index 5d5b44fb91ba..d7c2f46d9efb 100644 --- a/example/adversary/README.md +++ b/example/adversary/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Adversarial examples This demonstrates the concept of "adversarial examples" from [1] showing how to fool a well-trained CNN. diff --git a/example/autoencoder/README.md b/example/autoencoder/README.md index 960636cd7d59..9db075e680f0 100644 --- a/example/autoencoder/README.md +++ b/example/autoencoder/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Example of a Convolutional Autoencoder Autoencoder architectures are often used for unsupervised feature learning. This [link](http://ufldl.stanford.edu/tutorial/unsupervised/Autoencoders/) contains an introduction tutorial to autoencoders. This example illustrates a simple autoencoder using a stack of convolutional layers for both the encoder and the decoder. diff --git a/example/autoencoder/variational_autoencoder/README.md b/example/autoencoder/variational_autoencoder/README.md index c6e68d54c4f9..2a7a4966f596 100644 --- a/example/autoencoder/variational_autoencoder/README.md +++ b/example/autoencoder/variational_autoencoder/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Variational Auto Encoder(VAE) ============================= diff --git a/example/bayesian-methods/README.md b/example/bayesian-methods/README.md index fc35b94219d7..7604495eb151 100644 --- a/example/bayesian-methods/README.md +++ b/example/bayesian-methods/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Bayesian Methods ================ diff --git a/example/bi-lstm-sort/README.md b/example/bi-lstm-sort/README.md index f00cc85caa30..e26e2b02de63 100644 --- a/example/bi-lstm-sort/README.md +++ b/example/bi-lstm-sort/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Bidirectionnal LSTM to sort an array. This is an example of using bidirectionmal lstm to sort an array. Please refer to the notebook. diff --git a/example/caffe/README.md b/example/caffe/README.md index 466305cc9b88..a497176a996f 100644 --- a/example/caffe/README.md +++ b/example/caffe/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # How to use Caffe operator in MXNet [Caffe](http://caffe.berkeleyvision.org/) has been a well-known and widely-used deep learning framework. Now MXNet has supported calling most caffe operators(layers) and loss functions directly in its symbolic graph! Using one's own customized caffe layer is also effortless. diff --git a/example/capsnet/README.md b/example/capsnet/README.md index 500c7df72515..9319ea3b43bc 100644 --- a/example/capsnet/README.md +++ b/example/capsnet/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + **CapsNet-MXNet** ========================================= diff --git a/example/captcha/README.md b/example/captcha/README.md index cc97442f6207..7743997e90bd 100644 --- a/example/captcha/README.md +++ b/example/captcha/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + This is the R version of [captcha recognition](http://blog.xlvector.net/2016-05/mxnet-ocr-cnn/) example by xlvector and it can be used as an example of multi-label training. For a captcha below, we consider it as an image with 4 labels and train a CNN over the data set. ![](captcha_example.png) diff --git a/example/captcha/mxnet_captcha.R b/example/captcha/mxnet_captcha.R index 8988d25104d5..43e819f8c264 100644 --- a/example/captcha/mxnet_captcha.R +++ b/example/captcha/mxnet_captcha.R @@ -39,9 +39,9 @@ label <- mx.symbol.Reshape(data = label, target_shape = c(0)) captcha_net <- mx.symbol.SoftmaxOutput(data = fc2, label = label, name = "softmax") mx.metric.acc2 <- mx.metric.custom("accuracy", function(label, pred) { - ypred <- max.col(t(pred)) - 1 + ypred <- max.col(t(data.matrix(pred))) - 1 ypred <- matrix(ypred, nrow = nrow(label), ncol = ncol(label), byrow = TRUE) - return(sum(colSums(label == ypred) == 4) / ncol(label)) + return(sum(colSums(data.matrix(label) == ypred) == 4) / ncol(label)) }) data.shape <- c(80, 30, 3) diff --git a/example/cnn_chinese_text_classification/README.md b/example/cnn_chinese_text_classification/README.md index bfb271dd5c45..e28a0ec9ac47 100644 --- a/example/cnn_chinese_text_classification/README.md +++ b/example/cnn_chinese_text_classification/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Implementing CNN + Highway Network for Chinese Text Classification in MXNet ============ Sentiment classification forked from [incubator-mxnet/cnn_text_classification/](https://github.com/apache/incubator-mxnet/tree/master/example/cnn_text_classification), i've implemented the [Highway Networks](https://arxiv.org/pdf/1505.00387.pdf) architecture.The final train model is CNN + Highway Network structure, and this version can achieve a best dev accuracy of 94.75% with the Chinese corpus. diff --git a/example/cnn_text_classification/README.md b/example/cnn_text_classification/README.md index 2f1991f319ac..f4ebc43afa96 100644 --- a/example/cnn_text_classification/README.md +++ b/example/cnn_text_classification/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Implementing CNN for Text Classification in MXNet ============ It is a slightly simplified implementation of Kim's [Convolutional Neural Networks for Sentence Classification](http://arxiv.org/abs/1408.5882) paper in MXNet. diff --git a/example/ctc/README.md b/example/ctc/README.md index a2f54cffaf86..e64c6e633b2f 100644 --- a/example/ctc/README.md +++ b/example/ctc/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Connectionist Temporal Classification [Connectionist Temporal Classification](https://www.cs.toronto.edu/~graves/icml_2006.pdf) (CTC) is a cost function that is used to train Recurrent Neural Networks (RNNs) to label unsegmented input sequence data in supervised learning. For example in a speech recognition application, using a typical cross-entropy loss the input signal needs to be segmented into words or sub-words. However, using CTC-loss, a single unaligned label sequence per input sequence is sufficient for the network to learn both the alignment and labeling. Baidu's warp-ctc page contains a more detailed [introduction to CTC-loss](https://github.com/baidu-research/warp-ctc#introduction). diff --git a/example/deep-embedded-clustering/README.md b/example/deep-embedded-clustering/README.md index 3972f90bda4a..f3b532d0023d 100644 --- a/example/deep-embedded-clustering/README.md +++ b/example/deep-embedded-clustering/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # DEC Implementation This is based on the paper `Unsupervised deep embedding for clustering analysis` by Junyuan Xie, Ross Girshick, and Ali Farhadi diff --git a/example/distributed_training/README.md b/example/distributed_training/README.md index b0b0447725b5..d1bd11738cec 100644 --- a/example/distributed_training/README.md +++ b/example/distributed_training/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Distributed Training using Gluon Deep learning models are usually trained using GPUs because GPUs can do a lot more computations in parallel that CPUs. But even with the modern GPUs, it could take several days to train big models. Training can be done faster by using multiple GPUs like described in [this](https://gluon.mxnet.io/chapter07_distributed-learning/multiple-gpus-gluon.html) tutorial. However only a certain number of GPUs can be attached to one host (typically 8 or 16). To make the training even faster, we can use multiple GPUs attached to multiple hosts. diff --git a/example/dsd/README.md b/example/dsd/README.md index 0ce5cc5d1f0f..4a2a02cddbfa 100644 --- a/example/dsd/README.md +++ b/example/dsd/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + DSD Training ============ This folder contains an optimizer class that implements DSD training coupled with SGD. The training diff --git a/example/fcn-xs/README.md b/example/fcn-xs/README.md index 49c57fc08eaf..2d81f71ed68b 100644 --- a/example/fcn-xs/README.md +++ b/example/fcn-xs/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + FCN-xs EXAMPLE -------------- This folder contains an example implementation for Fully Convolutional Networks (FCN) in MXNet. diff --git a/example/gan/CGAN_mnist_R/README.md b/example/gan/CGAN_mnist_R/README.md new file mode 100644 index 000000000000..bf0bb08b1147 --- /dev/null +++ b/example/gan/CGAN_mnist_R/README.md @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + +# Conditional Generative Adversarial Network with MXNet R package + +This tutorial shows how to build and train a Conditional Generative Adversarial Network (CGAN) on MNIST images. + +## How GAN works +A Generative Adversarial Model simultaneously trains two models: a generator that learns to output fake samples from an unknown distribution and a discriminator that learns to distinguish fake from real samples. + +The CGAN is a conditional variation of the GAN where the generator is instructed to generate a real sample having specific characteristics rather than a generic sample from full distribution. Such condition could be the label associated with an image like in this tutorial or a more detailed tag as shown in the example below: + +![Image credit: (Scott Reed)[https://github.com/reedscot/icml2016]](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/dcgan_network.jpg) + +## Initial setup + +The following packages are needed to run the tutorial: + +``` +require("imager") +require("dplyr") +require("readr") +require("mxnet") +``` + +The full demo is comprised of the two following scripts: + +```CGAN_mnist_setup.R```: prepare data and define the model structure +```CGAN_train.R```: execute the training + +## Data preperation + +The MNIST dataset is available [here](https://www.kaggle.com/c/digit-recognizer/data)). Once train.csv is downloaded into the data/ folder, we can import into R. + +```train <- read_csv('data/train.csv') +train <- data.matrix(train) + +train_data <- train[,-1] +train_data <- t(train_data/255*2-1) +train_label <- as.integer(train[,1]) + +dim(train_data) <- c(28, 28, 1, ncol(train_data)) +``` +Custom iterators are defined in ```iterators.R``` and imported by ```CGAN_mnist_setup.R``` + +## Generator +The generator is a network that creates novel samples (MNIST images) from 2 inputs: + +- Noise vector +- Labels defining the object condition (which digit to produce) + +The noise vector provides the building blocks to the Generator model, which will learns how to structure that noise into a sample. The mx.symbol.Deconvolution operator is used to upsample the initial input from a 1x1 shape up to a 28x28 image. + +The information on the label for which to generate a fake sample is provided by a one-hot encoding of the label indices that is appended to the random noise. For MNIST, the 0-9 indices are therefore converted into a binary vector of length 10. More complex applications would require embeddings rather than simple one-hot to encode the condition. + +## Discriminator +The discriminator attempts to distinguish between fake samples produced by the generator and real ones sampled from MNIST training data. + +In a conditional GAN, the labels associated with the samples are also provided to the Discriminator. In this demo, this information is again provided as a hot-hot encoding of the label that is broadcast to match the image dimensions (10 -> 28x28x10). + +## Training logic +The training process of the discriminator is most obvious: the loss is simple a binary TRUE/FALSE response and that loss is propagated back into the CNN network. It can therefore be understood as a simple binary classification problem. + +```### Train loop on fake +mx.exec.update.arg.arrays(exec_D, arg.arrays = + list(data=D_data_fake, digit=D_digit_fake, label=mx.nd.array(rep(0, batch_size))), + match.name=TRUE) +mx.exec.forward(exec_D, is.train=T) +mx.exec.backward(exec_D) +update_args_D<- updater_D(weight = exec_D$ref.arg.arrays, grad = exec_D$ref.grad.arrays) +mx.exec.update.arg.arrays(exec_D, update_args_D, skip.null=TRUE) + +### Train loop on real +mx.exec.update.arg.arrays(exec_D, arg.arrays = + list(data=D_data_real, digit=D_digit_real, label=mx.nd.array(rep(1, batch_size))), + match.name=TRUE) +mx.exec.forward(exec_D, is.train=T) +mx.exec.backward(exec_D) +update_args_D<- updater_D(weight = exec_D$ref.arg.arrays, grad = exec_D$ref.grad.arrays) +mx.exec.update.arg.arrays(exec_D, update_args_D, skip.null=TRUE) +``` + +The generator loss comes from the backpropagation of the the discriminator loss into its generated output. By faking the generator labels to be real samples into the discriminator, the discriminator back-propagated loss provides the generator with the information on how to best adapt its parameters to trick the discriminator into believing the fake samples are real. + +This requires to backpropagate the gradients up to the input data of the discriminator (whereas this input gradient is typically ignored in vanilla feedforward network). + +```### Update Generator weights - use a seperate executor for writing data gradients +exec_D_back <- mxnet:::mx.symbol.bind(symbol = D_sym, + arg.arrays = exec_D$arg.arrays, + aux.arrays = exec_D$aux.arrays, grad.reqs = rep("write", length(exec_D$arg.arrays)), + ctx = devices) + +mx.exec.update.arg.arrays(exec_D_back, arg.arrays = + list(data=D_data_fake, digit=D_digit_fake, label=mx.nd.array(rep(1, batch_size))), + match.name=TRUE) +mx.exec.forward(exec_D_back, is.train=T) +mx.exec.backward(exec_D_back) +D_grads <- exec_D_back$ref.grad.arrays$data +mx.exec.backward(exec_G, out_grads=D_grads) + +update_args_G <- updater_G(weight = exec_G$ref.arg.arrays, grad = exec_G$ref.grad.arrays) +mx.exec.update.arg.arrays(exec_G, update_args_G, skip.null=TRUE) +``` + +The above training steps are executed in the ```CGAN_train.R``` script. + +## Monitor the training + +During training, the [imager](http://dahtah.github.io/imager/) package facilitates the visual quality assessment of the fake samples. + +```if (iteration==1 | iteration %% 100==0){ + par(mfrow=c(3,3), mar=c(0.1,0.1,0.1,0.1)) + for (i in 1:9) { + img <- as.array(exec_G$ref.outputs$G_sym_output)[,,,i] + plot(as.cimg(img), axes=F) + } +} +``` +Below are samples obtained at different stage of the training. + +Starting from noise: + +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/CGAN_1.png) + +Slowly getting it - iteration 200: + +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/CGAN_200.png) + +Generate specified digit images on demand - iteration 2400: + +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/CGAN_2400.png) + +## Inference + +Once the model is trained, synthetic images of the desired digit can be produced by feeding the generator with fixed labels rather than the randomly generated ones used during the training. + +Here we will generate fake ```9```: + +```digit <- mx.nd.array(rep(9, times=batch_size)) +data <- mx.nd.one.hot(indices = digit, depth = 10) +data <- mx.nd.reshape(data = data, shape = c(1,1,-1, batch_size)) + +exec_G <- mx.simple.bind(symbol = G_sym, data=data_shape_G, ctx = devices, grad.req = "null") +mx.exec.update.arg.arrays(exec_G, G_arg_params, match.name=TRUE) +mx.exec.update.arg.arrays(exec_G, list(data=data), match.name=TRUE) +mx.exec.update.aux.arrays(exec_G, G_aux_params, match.name=TRUE) + +mx.exec.forward(exec_G, is.train=F) +``` +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gan/CGAN_mnist_R/CGAN_infer_9.png) + +Further details of the CGAN methodology can be found in the paper [Generative Adversarial Text to Image Synthesis](https://arxiv.org/abs/1605.05396). + + diff --git a/example/gluon/actor_critic/README.md b/example/gluon/actor_critic/README.md new file mode 100644 index 000000000000..7f3a6a73e972 --- /dev/null +++ b/example/gluon/actor_critic/README.md @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + +# Actor Critic Model + +This example shows an actor critic model that consists of a critic that measures how good an action taken is and an actor that controls the agent's behavior. +In our example actor and critic use the same model: + +``` +class Policy(gluon.Block): + def __init__(self, **kwargs): + super(Policy, self).__init__(**kwargs) + with self.name_scope(): + self.dense = nn.Dense(16, in_units=4, activation='relu') + self.action_pred = nn.Dense(2, in_units=16) + self.value_pred = nn.Dense(1, in_units=16) + + def forward(self, x): + x = self.dense(x) + probs = self.action_pred(x) + values = self.value_pred(x) + return F.softmax(probs), values +``` +The example uses [Gym](https://gym.openai.com/docs/), which is a toolkit for developing and comparing reinforcement learning algorithms. The model is running an instance of [CartPole-v0](https://gym.openai.com/envs/CartPole-v0/) that simulates a pole that is attached by an un-actuated joint to a cart, which moves along a frictionless track. The goal is to prevent it from falling over. + + +The example provides the following commandline options: +``` +MXNet actor-critic example + +optional arguments: + -h, --help show this help message and exit + --gamma G discount factor (default: 0.99) + --seed N random seed (default: 1) + --render render the environment + --log-interval N interval between training status logs (default: 10) + +``` + +To run the model execute, type +``` +python actor_critic.py --render +``` + +You will get an output like the following: +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gluon/actor_critic/actor_critic.gif) + diff --git a/example/gluon/audio/README.md b/example/gluon/audio/README.md new file mode 100644 index 000000000000..cb2b53eb3b83 --- /dev/null +++ b/example/gluon/audio/README.md @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + +# Urban Sounds Classification in MXNet Gluon + +This example provides an end-to-end pipeline for a common datahack competition - [Urban Sounds Classification Example](https://datahack.analyticsvidhya.com/contest/practice-problem-urban-sound-classification/). + +After logging in, the data set can be downloaded. +The details of the dataset and the link to download it are given below: + + +## Urban Sounds Dataset: +### Description + The dataset contains 8732 wav files which are audio samples(<= 4s)) of street sounds like engine_idling, car_horn, children_playing, dog_barking and so on. + The task is to classify these audio samples into one of the following 10 labels: + ``` + siren, + street_music, + drilling, + dog_bark, + children_playing, + gun_shot, + engine_idling, + air_conditioner, + jackhammer, + car_horn + ``` + +To be able to run this example: + +1. `pip install -r requirements.txt` + + If you are in the directory where the requirements.txt file lies, + this step installs the required libraries to run the example. + The main dependency that is required is: Librosa. + The version used to test the example is: `0.6.2` + For more details, refer here: +https://librosa.github.io/librosa/install.html + +2. Download the dataset(train.zip, test.zip) required for this example from the location: +https://drive.google.com/drive/folders/0By0bAi7hOBAFUHVXd1JCN3MwTEU + +3. Extract both the zip archives into the **current directory** - after unzipping you would get 2 new folders namely, + **Train** and **Test** and two csv files - **train.csv**, **test.csv** + + Assuming you are in a directory *"UrbanSounds"*, after downloading and extracting train.zip, the folder structure should be: + + ``` + UrbanSounds + - Train + - 0.wav, 1.wav ... + - train.csv + - train.py + - predict.py ... + ``` + +4. Apache MXNet is installed on the machine. For instructions, go to the link: https://mxnet.incubator.apache.org/install/ + + + +For information on the current design of how the AudioFolderDataset is implemented, refer below: +https://cwiki.apache.org/confluence/display/MXNET/Gluon+-+Audio + +### Usage + +For training: + +- Arguments + - train : The folder/directory that contains the audio(wav) files locally. Default = "./Train" + - csv: The file name of the csv file that contains audio file name to label mapping. Default = "train.csv" + - epochs : Number of epochs to train the model. Default = 30 + - batch_size : The batch size for training. Default = 32 + + +###### To use the default arguments, use: +``` +python train.py +``` +or + +###### To pass command-line arguments for training data directory, epochs, batch_size, csv file name, use : +``` +python train.py --train ./Train --csv train.csv --batch_size 32 --epochs 30 +``` + +For prediction: + +- Arguments + - pred : The folder/directory that contains the audio(wav) files which are to be classified. Default = "./Test" + + +###### To use the default arguments, use: +``` +python predict.py +``` +or + +###### To pass command-line arguments for test data directory, use : +``` +python predict.py --pred ./Test +``` diff --git a/example/gluon/audio/urban_sounds/README.md b/example/gluon/audio/urban_sounds/README.md index c85d29db2e5a..c593f28dbee0 100644 --- a/example/gluon/audio/urban_sounds/README.md +++ b/example/gluon/audio/urban_sounds/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Urban Sounds Classification in MXNet Gluon This example provides an end-to-end pipeline for a common datahack competition - Urban Sounds Classification Example. diff --git a/example/gluon/dc_gan/README.md b/example/gluon/dc_gan/README.md index 5aacd78a3ed5..fd41d198a69d 100644 --- a/example/gluon/dc_gan/README.md +++ b/example/gluon/dc_gan/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # DCGAN in MXNet [Deep Convolutional Generative Adversarial Networks(DCGAN)](https://arxiv.org/abs/1511.06434) implementation with Apache MXNet GLUON. diff --git a/example/gluon/embedding_learning/README.md b/example/gluon/embedding_learning/README.md index e7821619a381..ce1fb536a859 100644 --- a/example/gluon/embedding_learning/README.md +++ b/example/gluon/embedding_learning/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Image Embedding Learning This example implements embedding learning based on a Margin-based Loss with distance weighted sampling [(Wu et al, 2017)](http://www.philkr.net/papers/2017-10-01-iccv/2017-10-01-iccv.pdf). The model obtains a validation Recall@1 of ~64% on the [Caltech-UCSD Birds-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html) dataset. diff --git a/example/gluon/house_prices/README.md b/example/gluon/house_prices/README.md new file mode 100644 index 000000000000..1393a0e3869d --- /dev/null +++ b/example/gluon/house_prices/README.md @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + +# House Prices: Advanced Regression Techniques + +This example shows how to predict house prices and it is based on the [House Price Kaggle challenge](https://www.kaggle.com/c/house-prices-advanced-regression-techniques#description) + +First you need to download train and test data set from here: +``` +https://www.kaggle.com/c/house-prices-advanced-regression-techniques/download/train.csv +https://www.kaggle.com/c/house-prices-advanced-regression-techniques/download/test.csv +``` +Afterwards you can execute the script with ```python kaggle_k_fold_cross_validation.py``` + +For a detailed explanation of the code, you can check out this [chapter](http://d2l.ai/chapter_deep-learning-basics/kaggle-house-price.html) of the Dive into Deep Learning book. diff --git a/example/gluon/lipnet/.gitignore b/example/gluon/lipnet/.gitignore new file mode 100644 index 000000000000..9a6ee993b157 --- /dev/null +++ b/example/gluon/lipnet/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +utils/*.dat + diff --git a/example/gluon/lipnet/BeamSearch.py b/example/gluon/lipnet/BeamSearch.py new file mode 100644 index 000000000000..1b41bc0020d1 --- /dev/null +++ b/example/gluon/lipnet/BeamSearch.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module : this module to decode using beam search +https://github.com/ThomasDelteil/HandwrittenTextRecognition_MXNet/blob/master/utils/CTCDecoder/BeamSearch.py +""" + +from __future__ import division +from __future__ import print_function +import numpy as np + +class BeamEntry: + """ + information about one single beam at specific time-step + """ + def __init__(self): + self.prTotal = 0 # blank and non-blank + self.prNonBlank = 0 # non-blank + self.prBlank = 0 # blank + self.prText = 1 # LM score + self.lmApplied = False # flag if LM was already applied to this beam + self.labeling = () # beam-labeling + +class BeamState: + """ + information about the beams at specific time-step + """ + def __init__(self): + self.entries = {} + + def norm(self): + """ + length-normalise LM score + """ + for (k, _) in self.entries.items(): + labelingLen = len(self.entries[k].labeling) + self.entries[k].prText = self.entries[k].prText ** (1.0 / (labelingLen if labelingLen else 1.0)) + + def sort(self): + """ + return beam-labelings, sorted by probability + """ + beams = [v for (_, v) in self.entries.items()] + sortedBeams = sorted(beams, reverse=True, key=lambda x: x.prTotal*x.prText) + return [x.labeling for x in sortedBeams] + +def applyLM(parentBeam, childBeam, classes, lm): + """ + calculate LM score of child beam by taking score from parent beam and bigram probability of last two chars + """ + if lm and not childBeam.lmApplied: + c1 = classes[parentBeam.labeling[-1] if parentBeam.labeling else classes.index(' ')] # first char + c2 = classes[childBeam.labeling[-1]] # second char + lmFactor = 0.01 # influence of language model + bigramProb = lm.getCharBigram(c1, c2) ** lmFactor # probability of seeing first and second char next to each other + childBeam.prText = parentBeam.prText * bigramProb # probability of char sequence + childBeam.lmApplied = True # only apply LM once per beam entry + +def addBeam(beamState, labeling): + """ + add beam if it does not yet exist + """ + if labeling not in beamState.entries: + beamState.entries[labeling] = BeamEntry() + +def ctcBeamSearch(mat, classes, lm, k, beamWidth): + """ + beam search as described by the paper of Hwang et al. and the paper of Graves et al. + """ + + blankIdx = len(classes) + maxT, maxC = mat.shape + + # initialise beam state + last = BeamState() + labeling = () + last.entries[labeling] = BeamEntry() + last.entries[labeling].prBlank = 1 + last.entries[labeling].prTotal = 1 + + # go over all time-steps + for t in range(maxT): + curr = BeamState() + + # get beam-labelings of best beams + bestLabelings = last.sort()[0:beamWidth] + + # go over best beams + for labeling in bestLabelings: + + # probability of paths ending with a non-blank + prNonBlank = 0 + # in case of non-empty beam + if labeling: + # probability of paths with repeated last char at the end + try: + prNonBlank = last.entries[labeling].prNonBlank * mat[t, labeling[-1]] + except FloatingPointError: + prNonBlank = 0 + + # probability of paths ending with a blank + prBlank = (last.entries[labeling].prTotal) * mat[t, blankIdx] + + # add beam at current time-step if needed + addBeam(curr, labeling) + + # fill in data + curr.entries[labeling].labeling = labeling + curr.entries[labeling].prNonBlank += prNonBlank + curr.entries[labeling].prBlank += prBlank + curr.entries[labeling].prTotal += prBlank + prNonBlank + curr.entries[labeling].prText = last.entries[labeling].prText # beam-labeling not changed, therefore also LM score unchanged from + curr.entries[labeling].lmApplied = True # LM already applied at previous time-step for this beam-labeling + + # extend current beam-labeling + for c in range(maxC - 1): + # add new char to current beam-labeling + newLabeling = labeling + (c,) + + # if new labeling contains duplicate char at the end, only consider paths ending with a blank + if labeling and labeling[-1] == c: + prNonBlank = mat[t, c] * last.entries[labeling].prBlank + else: + prNonBlank = mat[t, c] * last.entries[labeling].prTotal + + # add beam at current time-step if needed + addBeam(curr, newLabeling) + + # fill in data + curr.entries[newLabeling].labeling = newLabeling + curr.entries[newLabeling].prNonBlank += prNonBlank + curr.entries[newLabeling].prTotal += prNonBlank + + # apply LM + applyLM(curr.entries[labeling], curr.entries[newLabeling], classes, lm) + + # set new beam state + last = curr + + # normalise LM scores according to beam-labeling-length + last.norm() + + # sort by probability + bestLabelings = last.sort()[:k] # get most probable labeling + + output = [] + for bestLabeling in bestLabelings: + # map labels to chars + res = '' + for l in bestLabeling: + res += classes[l] + output.append(res) + return output \ No newline at end of file diff --git a/example/gluon/lipnet/README.md b/example/gluon/lipnet/README.md new file mode 100644 index 000000000000..89c27a11330f --- /dev/null +++ b/example/gluon/lipnet/README.md @@ -0,0 +1,254 @@ + + +# LipNet: End-to-End Sentence-level Lipreading + +--- + +This is a Gluon implementation of [LipNet: End-to-End Sentence-level Lipreading](https://arxiv.org/abs/1611.01599) + +![net_structure](asset/network_structure.png) + +![sample output](https://user-images.githubusercontent.com/11376047/52533982-d7227680-2d7e-11e9-9f18-c15b952faf0e.png) + +## Requirements +- Python 3.6.4 +- MXNet 1.3.0 +- Required disk space: 35 GB +``` +pip install -r requirements.txt +``` + +--- + +## The Data +- The GRID audiovisual sentence corpus (http://spandh.dcs.shef.ac.uk/gridcorpus/) + - GRID is a large multi-talker audiovisual sentence corpus to support joint computational-behavioral studies in speech perception. In brief, the corpus consists of high-quality audio and video (facial) recordings of 1000 sentences spoken by each of 34 talkers (18 male, 16 female). Sentences are of the form "put red at G9 now". The corpus, together with transcriptions, is freely available for research use. +- Video: (normal)(480 M each) + - Each movie has one sentence consist of 6 words. +- Align: word alignments (190 K each) + - One align has 6 words. Each word has start time and end time. But this tutorial needs just sentence because of using ctc-loss. + +--- + +## Pretrained model +You can train the model yourself in the following sections, you can test a pretrained model's inference, or resume training from the model checkpoint. To work with the provided pretrained model, first download it, then run one of the provided Python scripts for inference (infer.py) or training (main.py). + +* Download the [pretrained model](https://github.com/soeque1/temp_files/files/2848870/epoches_81_loss_15.7157.zip) +* Try inference with the following: + +``` +python infer.py model_path='checkpoint/epoches_81_loss_15.7157' +``` + +* Resume training with the following: + +``` +python main.py model_path='checkpoint/epoches_81_loss_15.7157' +``` + +## Prepare the Data + +You can prepare the data yourself, or you can download preprocessed data. + +### Option 1 - Download the preprocessed data + +There are two download routes provided for the preprocessed data. + +#### Download and untar the data +To download tar zipped files by link, download the following files and extract in a folder called `data` in the root of this example folder. You should have the following structure: +``` +/lipnet/data/align +/lipnet/data/datasets +``` + +* [align files](https://mxnet-public.s3.amazonaws.com/lipnet/data-archives/align.tgz) +* [datasets files](https://mxnet-public.s3.amazonaws.com/lipnet/data-archives/datasets.tgz) + +#### Use AWS CLI to sync the data +To get the folders and files all unzipped with AWS CLI, can use the following command. This will provide the folder structure for you. Run this command from `/lipnet/`: + +``` + aws s3 sync s3://mxnet-public/lipnet/data . +``` + +### Option 2 (part 1)- Download the raw dataset +- Outputs + - The Total Movies(mp4): 16GB + - The Total Aligns(text): 134MB +- Arguments + - src_path : Path for videos (default='./data/mp4s/') + - align_path : Path for aligns (default='./data/') + - n_process : num of process (default=1) + +``` +cd ./utils && python download_data.py --n_process=$(nproc) +``` + +### Option 2 (part 2) Preprocess the raw dataset: Extracting the mouth images from a video and save it + +* Using Face Landmark Detection(http://dlib.net/) + +#### Preprocess (preprocess_data.py) +* If there is no landmark, it download automatically. +* Using Face Landmark Detection, It extract the mouth from a video. + +- example: + - video: ./data/mp4s/s2/bbbf7p.mpg + - align(target): ./data/align/s2/bbbf7p.align + : 'sil bin blue by f seven please sil' + + +- Video to the images (75 Frames) + +Frame 0 | Frame 1 | ... | Frame 74 | +:-------------------------:|:-------------------------:|:-------------------------:|:-------------------------: +![](asset/s2_bbbf7p_000.png) | ![](asset/s2_bbbf7p_001.png) | ... | ![](asset/s2_bbbf7p_074.png) + + - Extract the mouth from images + +Frame 0 | Frame 1 | ... | Frame 74 | +:-------------------------:|:-------------------------:|:-------------------------:|:-------------------------: +![](asset/mouth_000.png) | ![](asset/mouth_001.png) | ... | ![](asset/mouth_074.png) + +* Save the result images into tgt_path. + +---- + +#### How to run the preprocess script + +- Arguments + - src_path : Path for videos (default='./data/mp4s/') + - tgt_path : Path for preprocessed images (default='./data/datasets/') + - n_process : num of process (default=1) + +- Outputs + - The Total Images(png): 19GB +- Elapsed time + - About 54 Hours using 1 process + - If you use the multi-processes, you can finish the number of processes faster. + - e.g) 9 hours using 6 processes + +You can run the preprocessing with just one processor, but this will take a long time (>48 hours). To use all of the available processors, use the following command: + +``` +cd ./utils && python preprocess_data.py --n_process=$(nproc) +``` + +#### Output: Data structure of the preprocessed data + +``` +The training data folder should look like : + + |--datasets + |--s1 + |--bbir7s + |--mouth_000.png + |--mouth_001.png + ... + |--bgaa8p + |--mouth_000.png + |--mouth_001.png + ... + |--s2 + ... + |--align + |--bw1d8a.align + |--bggzzs.align + ... + +``` + +--- + +## Training +After you have acquired the preprocessed data you are ready to train the lipnet model. + +- According to [LipNet: End-to-End Sentence-level Lipreading](https://arxiv.org/abs/1611.01599), four (S1, S2, S20, S22) of the 34 subjects are used for evaluation. + The other subjects are used for training. + +- To use the multi-gpu, it is recommended to make the batch size $(num_gpus) times larger. + + - e.g) 1-gpu and 128 batch_size > 2-gpus 256 batch_size + + +- arguments + - batch_size : Define batch size (default=64) + - epochs : Define total epochs (default=100) + - image_path : Path for lip image files (default='./data/datasets/') + - align_path : Path for align files (default='./data/align/') + - dr_rate : Dropout rate(default=0.5) + - num_gpus : Num of gpus (if num_gpus is 0, then use cpu) (default=1) + - num_workers : Num of workers when generating data (default=0) + - model_path : Path of pretrained model (default=None) + +``` +python main.py +``` + +--- + +## Test Environment +- 72 CPU cores +- 1 GPU (NVIDIA Tesla V100 SXM2 32 GB) +- 128 Batch Size + + - It takes over 24 hours (60 epochs) to get some good results. + +--- + +## Inference + +- arguments + - batch_size : Define batch size (default=64) + - image_path : Path for lip image files (default='./data/datasets/') + - align_path : Path for align files (default='./data/align/') + - num_gpus : Num of gpus (if num_gpus is 0, then use cpu) (default=1) + - num_workers : Num of workers when generating data (default=0) + - data_type : 'train' or 'valid' (defalut='valid') + - model_path : Path of pretrained model (default=None) + +``` +python infer.py --model_path=$(model_path) +``` + + +``` +[Target] +['lay green with a zero again', + 'bin blue with r nine please', + 'set blue with e five again', + 'bin green by t seven soon', + 'lay red at d five now', + 'bin green in x eight now', + 'bin blue with e one now', + 'lay red at j nine now'] + ``` + + ``` +[Pred] +['lay green with s zero again', + 'bin blue with r nine please', + 'set blue with e five again', + 'bin green by t seven soon', + 'lay red at c five now', + 'bin green in x eight now', + 'bin blue with m one now', + 'lay red at j nine now'] + ``` diff --git a/example/gluon/lipnet/asset/mouth_000.png b/example/gluon/lipnet/asset/mouth_000.png new file mode 100644 index 000000000000..b318e56dfd21 Binary files /dev/null and b/example/gluon/lipnet/asset/mouth_000.png differ diff --git a/example/gluon/lipnet/asset/mouth_001.png b/example/gluon/lipnet/asset/mouth_001.png new file mode 100644 index 000000000000..60bd04ab18ae Binary files /dev/null and b/example/gluon/lipnet/asset/mouth_001.png differ diff --git a/example/gluon/lipnet/asset/mouth_074.png b/example/gluon/lipnet/asset/mouth_074.png new file mode 100644 index 000000000000..e5e0d78e2450 Binary files /dev/null and b/example/gluon/lipnet/asset/mouth_074.png differ diff --git a/example/gluon/lipnet/asset/network_structure.png b/example/gluon/lipnet/asset/network_structure.png new file mode 100644 index 000000000000..eeec2cb0b645 Binary files /dev/null and b/example/gluon/lipnet/asset/network_structure.png differ diff --git a/example/gluon/lipnet/asset/s2_bbbf7p_000.png b/example/gluon/lipnet/asset/s2_bbbf7p_000.png new file mode 100644 index 000000000000..6495d2fa5b83 Binary files /dev/null and b/example/gluon/lipnet/asset/s2_bbbf7p_000.png differ diff --git a/example/gluon/lipnet/asset/s2_bbbf7p_001.png b/example/gluon/lipnet/asset/s2_bbbf7p_001.png new file mode 100644 index 000000000000..2a7e269f14de Binary files /dev/null and b/example/gluon/lipnet/asset/s2_bbbf7p_001.png differ diff --git a/example/gluon/lipnet/asset/s2_bbbf7p_074.png b/example/gluon/lipnet/asset/s2_bbbf7p_074.png new file mode 100644 index 000000000000..eabd392be49c Binary files /dev/null and b/example/gluon/lipnet/asset/s2_bbbf7p_074.png differ diff --git a/tests/python/unittest/test_libinfo.py b/example/gluon/lipnet/checkpoint/__init__.py similarity index 74% rename from tests/python/unittest/test_libinfo.py rename to example/gluon/lipnet/checkpoint/__init__.py index 66bf03111d4b..13a83393a912 100644 --- a/tests/python/unittest/test_libinfo.py +++ b/example/gluon/lipnet/checkpoint/__init__.py @@ -14,17 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -import os -import mxnet as mx -from mxnet import libinfo - -def test_include_path(): - incl_path = libinfo.find_include_path() - assert os.path.exists(incl_path) - assert os.path.isdir(incl_path) - - -if __name__ == '__main__': - import nose - nose.runmodule() diff --git a/example/gluon/lipnet/data_loader.py b/example/gluon/lipnet/data_loader.py new file mode 100644 index 000000000000..e3cc24bfcc63 --- /dev/null +++ b/example/gluon/lipnet/data_loader.py @@ -0,0 +1,94 @@ +""" +Description : Set DataSet module for lip images +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import glob +from mxnet import nd +import mxnet.gluon.data.dataset as dataset +from mxnet.gluon.data.vision.datasets import image +from utils.align import Align + +# pylint: disable=too-many-instance-attributes, too-many-arguments +class LipsDataset(dataset.Dataset): + """ + Description : DataSet class for lip images + """ + def __init__(self, root, align_root, flag=1, + mode='train', transform=None, seq_len=75): + assert mode in ['train', 'valid'] + self._root = os.path.expanduser(root) + self._align_root = align_root + self._flag = flag + self._transform = transform + self._exts = ['.jpg', '.jpeg', '.png'] + self._seq_len = seq_len + self._mode = mode + self._list_images(self._root) + + def _list_images(self, root): + """ + Description : generate list for lip images + """ + self.labels = [] + self.items = [] + + valid_unseen_sub_idx = [1, 2, 20, 22] + skip_sub_idx = [21] + + if self._mode == 'train': + sub_idx = ['s' + str(i) for i in range(1, 35) \ + if i not in valid_unseen_sub_idx + skip_sub_idx] + elif self._mode == 'valid': + sub_idx = ['s' + str(i) for i in valid_unseen_sub_idx] + + folder_path = [] + for i in sub_idx: + folder_path.extend(glob.glob(os.path.join(root, i, "*"))) + + for folder in folder_path: + filename = glob.glob(os.path.join(folder, "*")) + if len(filename) != self._seq_len: + continue + filename.sort() + label = os.path.split(folder)[-1] + self.items.append((filename, label)) + + def align_generation(self, file_nm, padding=75): + """ + Description : Align to lip position + """ + align = Align(self._align_root + '/' + file_nm + '.align') + return nd.array(align.sentence(padding)) + + def __getitem__(self, idx): + img = list() + for image_name in self.items[idx][0]: + tmp_img = image.imread(image_name, self._flag) + if self._transform is not None: + tmp_img = self._transform(tmp_img) + img.append(tmp_img) + img = nd.stack(*img) + img = nd.transpose(img, (1, 0, 2, 3)) + label = self.align_generation(self.items[idx][1], + padding=self._seq_len) + return img, label + + def __len__(self): + return len(self.items) diff --git a/example/gluon/lipnet/infer.py b/example/gluon/lipnet/infer.py new file mode 100644 index 000000000000..746df9a05e72 --- /dev/null +++ b/example/gluon/lipnet/infer.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Description : main module to run the lipnet inference code +""" + + +import argparse +from trainer import Train + +def main(): + """ + Description : run lipnet training code using argument info + """ + parser = argparse.ArgumentParser() + parser.add_argument('--batch_size', type=int, default=64) + parser.add_argument('--image_path', type=str, default='./data/datasets/') + parser.add_argument('--align_path', type=str, default='./data/align/') + parser.add_argument('--num_gpus', type=int, default=1) + parser.add_argument('--num_workers', type=int, default=0) + parser.add_argument('--data_type', type=str, default='valid') + parser.add_argument('--model_path', type=str, default=None) + config = parser.parse_args() + trainer = Train(config) + trainer.build_model(path=config.model_path) + trainer.load_dataloader() + + if config.data_type == 'train': + data_loader = trainer.train_dataloader + elif config.data_type == 'valid': + data_loader = trainer.valid_dataloader + + trainer.infer_batch(data_loader) + +if __name__ == "__main__": + main() + \ No newline at end of file diff --git a/example/gluon/lipnet/main.py b/example/gluon/lipnet/main.py new file mode 100644 index 000000000000..8e5e7569d271 --- /dev/null +++ b/example/gluon/lipnet/main.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Description : main module to run the lipnet training code +""" + + +import argparse +from trainer import Train + +def main(): + """ + Description : run lipnet training code using argument info + """ + parser = argparse.ArgumentParser() + parser.add_argument('--batch_size', type=int, default=64) + parser.add_argument('--epochs', type=int, default=100) + parser.add_argument('--image_path', type=str, default='./data/datasets/') + parser.add_argument('--align_path', type=str, default='./data/align/') + parser.add_argument('--dr_rate', type=float, default=0.5) + parser.add_argument('--num_gpus', type=int, default=1) + parser.add_argument('--num_workers', type=int, default=0) + parser.add_argument('--model_path', type=str, default=None) + config = parser.parse_args() + trainer = Train(config) + trainer.build_model(dr_rate=config.dr_rate, path=config.model_path) + trainer.load_dataloader() + trainer.run(epochs=config.epochs) + +if __name__ == "__main__": + main() + \ No newline at end of file diff --git a/example/gluon/lipnet/models/__init__.py b/example/gluon/lipnet/models/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/example/gluon/lipnet/models/network.py b/example/gluon/lipnet/models/network.py new file mode 100644 index 000000000000..b8f005a961c1 --- /dev/null +++ b/example/gluon/lipnet/models/network.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Description : LipNet module using gluon +""" + +from mxnet.gluon import nn, rnn +# pylint: disable=too-many-instance-attributes +class LipNet(nn.HybridBlock): + """ + Description : LipNet network using gluon + dr_rate : Dropout rate + """ + def __init__(self, dr_rate, **kwargs): + super(LipNet, self).__init__(**kwargs) + with self.name_scope(): + self.conv1 = nn.Conv3D(32, kernel_size=(3, 5, 5), strides=(1, 2, 2), padding=(1, 2, 2)) + self.bn1 = nn.InstanceNorm(in_channels=32) + self.dr1 = nn.Dropout(dr_rate, axes=(1, 2)) + self.pool1 = nn.MaxPool3D((1, 2, 2), (1, 2, 2)) + self.conv2 = nn.Conv3D(64, kernel_size=(3, 5, 5), strides=(1, 1, 1), padding=(1, 2, 2)) + self.bn2 = nn.InstanceNorm(in_channels=64) + self.dr2 = nn.Dropout(dr_rate, axes=(1, 2)) + self.pool2 = nn.MaxPool3D((1, 2, 2), (1, 2, 2)) + self.conv3 = nn.Conv3D(96, kernel_size=(3, 3, 3), strides=(1, 1, 1), padding=(1, 2, 2)) + self.bn3 = nn.InstanceNorm(in_channels=96) + self.dr3 = nn.Dropout(dr_rate, axes=(1, 2)) + self.pool3 = nn.MaxPool3D((1, 2, 2), (1, 2, 2)) + self.gru1 = rnn.GRU(256, bidirectional=True) + self.gru2 = rnn.GRU(256, bidirectional=True) + self.dense = nn.Dense(27+1, flatten=False) + + # pylint: disable=arguments-differ + def hybrid_forward(self, F, x): + out = self.conv1(x) + out = self.bn1(out) + out = F.relu(out) + out = self.dr1(out) + out = self.pool1(out) + out = self.conv2(out) + out = self.bn2(out) + out = F.relu(out) + out = self.dr2(out) + out = self.pool2(out) + out = self.conv3(out) + out = self.bn3(out) + out = F.relu(out) + out = self.dr3(out) + out = self.pool3(out) + out = F.transpose(out, (2, 0, 1, 3, 4)) + # pylint: disable=no-member + out = out.reshape((0, 0, -1)) + out = self.gru1(out) + out = self.gru2(out) + out = self.dense(out) + out = F.log_softmax(out, axis=2) + out = F.transpose(out, (1, 0, 2)) + return out diff --git a/example/gluon/lipnet/requirements.txt b/example/gluon/lipnet/requirements.txt new file mode 100644 index 000000000000..f1fcda31d98f --- /dev/null +++ b/example/gluon/lipnet/requirements.txt @@ -0,0 +1,7 @@ +dlib==19.15.0 +Pillow==4.1.0 +scipy==0.19.0 +scikit-image==0.13.1 +scikit-video==1.1.11 +sk-video==1.1.10 +tqdm diff --git a/example/gluon/lipnet/tests/test_beamsearch.py b/example/gluon/lipnet/tests/test_beamsearch.py new file mode 100644 index 000000000000..069cbaee8e7f --- /dev/null +++ b/example/gluon/lipnet/tests/test_beamsearch.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""it is the test for the decode using beam search +Ref: +https://github.com/ThomasDelteil/HandwrittenTextRecognition_MXNet/blob/master/utils/CTCDecoder/BeamSearch.py +""" + +import unittest +import numpy as np +from BeamSearch import ctcBeamSearch + +class TestBeamSearch(unittest.TestCase): + """Test Beam Search + """ + def test_ctc_beam_search(self): + "test decoder" + classes = 'ab' + mat = np.array([[0.4, 0, 0.6], [0.4, 0, 0.6]]) + print('Test beam search') + expected = 'a' + actual = ctcBeamSearch(mat, classes, None, k=2, beamWidth=3)[0] + print('Expected: "' + expected + '"') + print('Actual: "' + actual + '"') + self.assertEqual(expected, actual) + +if __name__ == '__main__': + unittest.main() diff --git a/example/gluon/lipnet/trainer.py b/example/gluon/lipnet/trainer.py new file mode 100644 index 000000000000..df5c86ece9b8 --- /dev/null +++ b/example/gluon/lipnet/trainer.py @@ -0,0 +1,232 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Description : Training module for LipNet +""" + + +import sys +import mxnet as mx +from mxnet import gluon, autograd, nd +from mxnet.gluon.data.vision import transforms +from tqdm import tqdm, trange +from data_loader import LipsDataset +from models.network import LipNet +from BeamSearch import ctcBeamSearch +from utils.common import char_conv, int2char +# set gpu count + + +def setting_ctx(num_gpus): + """ + Description : set gpu module + """ + if num_gpus > 0: + ctx = [mx.gpu(i) for i in range(num_gpus)] + else: + ctx = [mx.cpu()] + return ctx + + +ALPHABET = '' +for i in range(27): + ALPHABET += int2char(i) + +def char_beam_search(out): + """ + Description : apply beam search for prediction result + """ + out_conv = list() + for idx in range(out.shape[0]): + probs = out[idx] + prob = probs.softmax().asnumpy() + line_string_proposals = ctcBeamSearch(prob, ALPHABET, None, k=4, beamWidth=25) + out_conv.append(line_string_proposals[0]) + return out_conv + +# pylint: disable=too-many-instance-attributes, too-many-locals +class Train: + """ + Description : Train class for training network + """ + def __init__(self, config): + ##setting hyper-parameters + self.batch_size = config.batch_size + self.image_path = config.image_path + self.align_path = config.align_path + self.num_gpus = config.num_gpus + self.ctx = setting_ctx(self.num_gpus) + self.num_workers = config.num_workers + self.seq_len = 75 + + def build_model(self, dr_rate=0, path=None): + """ + Description : build network + """ + #set network + self.net = LipNet(dr_rate) + self.net.hybridize() + self.net.initialize(ctx=self.ctx) + + if path is not None: + self.load_model(path) + + #set optimizer + self.loss_fn = gluon.loss.CTCLoss() + self.trainer = gluon.Trainer(self.net.collect_params(), \ + optimizer='SGD') + + def save_model(self, epoch, loss): + """ + Description : save parameter of network weight + """ + prefix = 'checkpoint/epoches' + file_name = "{prefix}_{epoch}_loss_{l:.4f}".format(prefix=prefix, + epoch=str(epoch), + l=loss) + self.net.save_parameters(file_name) + + def load_model(self, path=''): + """ + Description : load parameter of network weight + """ + self.net.load_parameters(path) + + def load_dataloader(self): + """ + Description : Setup the dataloader + """ + + input_transform = transforms.Compose([transforms.ToTensor(), \ + transforms.Normalize((0.7136, 0.4906, 0.3283), \ + (0.1138, 0.1078, 0.0917))]) + training_dataset = LipsDataset(self.image_path, + self.align_path, + mode='train', + transform=input_transform, + seq_len=self.seq_len) + + self.train_dataloader = mx.gluon.data.DataLoader(training_dataset, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers) + + valid_dataset = LipsDataset(self.image_path, + self.align_path, + mode='valid', + transform=input_transform, + seq_len=self.seq_len) + + self.valid_dataloader = mx.gluon.data.DataLoader(valid_dataset, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers) + + def train(self, data, label, batch_size): + """ + Description : training for LipNet + """ + # pylint: disable=no-member + sum_losses = 0 + len_losses = 0 + with autograd.record(): + losses = [self.loss_fn(self.net(X), Y) for X, Y in zip(data, label)] + for loss in losses: + sum_losses += mx.nd.array(loss).sum().asscalar() + len_losses += len(loss) + loss.backward() + self.trainer.step(batch_size) + return sum_losses, len_losses + + def infer(self, input_data, input_label): + """ + Description : Print sentence for prediction result + """ + sum_losses = 0 + len_losses = 0 + for data, label in zip(input_data, input_label): + pred = self.net(data) + sum_losses += mx.nd.array(self.loss_fn(pred, label)).sum().asscalar() + len_losses += len(data) + pred_convert = char_beam_search(pred) + label_convert = char_conv(label.asnumpy()) + for target, pred in zip(label_convert, pred_convert): + print("target:{t} pred:{p}".format(t=target, p=pred)) + return sum_losses, len_losses + + def train_batch(self, dataloader): + """ + Description : training for LipNet + """ + sum_losses = 0 + len_losses = 0 + for input_data, input_label in tqdm(dataloader): + data = gluon.utils.split_and_load(input_data, self.ctx, even_split=False) + label = gluon.utils.split_and_load(input_label, self.ctx, even_split=False) + batch_size = input_data.shape[0] + sum_losses, len_losses = self.train(data, label, batch_size) + sum_losses += sum_losses + len_losses += len_losses + + return sum_losses, len_losses + + def infer_batch(self, dataloader): + """ + Description : inference for LipNet + """ + sum_losses = 0 + len_losses = 0 + for input_data, input_label in dataloader: + data = gluon.utils.split_and_load(input_data, self.ctx, even_split=False) + label = gluon.utils.split_and_load(input_label, self.ctx, even_split=False) + sum_losses, len_losses = self.infer(data, label) + sum_losses += sum_losses + len_losses += len_losses + + return sum_losses, len_losses + + def run(self, epochs): + """ + Description : Run training for LipNet + """ + best_loss = sys.maxsize + for epoch in trange(epochs): + iter_no = 0 + + ## train + sum_losses, len_losses = self.train_batch(self.train_dataloader) + + if iter_no % 20 == 0: + current_loss = sum_losses / len_losses + print("[Train] epoch:{e} iter:{i} loss:{l:.4f}".format(e=epoch, + i=iter_no, + l=current_loss)) + + ## validating + sum_val_losses, len_val_losses = self.infer_batch(self.valid_dataloader) + + current_val_loss = sum_val_losses / len_val_losses + print("[Vaild] epoch:{e} iter:{i} loss:{l:.4f}".format(e=epoch, + i=iter_no, + l=current_val_loss)) + + if best_loss > current_val_loss: + self.save_model(epoch, current_val_loss) + best_loss = current_val_loss + + iter_no += 1 diff --git a/example/gluon/lipnet/utils/__init__.py b/example/gluon/lipnet/utils/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/example/gluon/lipnet/utils/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/example/gluon/lipnet/utils/align.py b/example/gluon/lipnet/utils/align.py new file mode 100644 index 000000000000..48d0716aaedd --- /dev/null +++ b/example/gluon/lipnet/utils/align.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: align +This is used when the data is genrated by LipsDataset +""" + +import numpy as np +from .common import word_to_vector + + +class Align(object): + """ + Preprocess for Align + """ + skip_list = ['sil', 'sp'] + + def __init__(self, align_path): + self.build(align_path) + + def build(self, align_path): + """ + Build the align array + """ + file = open(align_path, 'r') + lines = file.readlines() + file.close() + # words: list([op, ed, word]) + words = [] + for line in lines: + _op, _ed, word = line.strip().split(' ') + if word not in Align.skip_list: + words.append((int(_op), int(_ed), word)) + self.words = words + self.n_words = len(words) + self.sentence_str = " ".join([w[2] for w in self.words]) + self.sentence_length = len(self.sentence_str) + + def sentence(self, padding=75): + """ + Get sentence + """ + vec = word_to_vector(self.sentence_str) + vec += [-1] * (padding - self.sentence_length) + return np.array(vec, dtype=np.int32) + + def word(self, _id, padding=75): + """ + Get words + """ + word = self.words[_id][2] + vec = word_to_vector(word) + vec += [-1] * (padding - len(vec)) + return np.array(vec, dtype=np.int32) + + def word_length(self, _id): + """ + Get the length of words + """ + return len(self.words[_id][2]) + + def word_frame_pos(self, _id): + """ + Get the position of words + """ + left = int(self.words[_id][0]/1000) + right = max(left+1, int(self.words[_id][1]/1000)) + return (left, right) diff --git a/example/gluon/lipnet/utils/common.py b/example/gluon/lipnet/utils/common.py new file mode 100644 index 000000000000..ec96b6879653 --- /dev/null +++ b/example/gluon/lipnet/utils/common.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: This module contains common conversion functions + +""" + + +def char2int(char): + """ + Convert character to integer. + """ + if char >= 'a' and char <= 'z': + return ord(char) - ord('a') + elif char == ' ': + return 26 + return None + + +def int2char(num): + """ + Convert integer to character. + """ + if num >= 0 and num < 26: + return chr(num + ord('a')) + elif num == 26: + return ' ' + return None + + +def word_to_vector(word): + """ + Convert character vectors to integer vectors. + """ + vector = [] + for char in list(word): + vector.append(char2int(char)) + return vector + + +def vector_to_word(vector): + """ + Convert integer vectors to character vectors. + """ + word = "" + for vec in vector: + word = word + int2char(vec) + return word + + +def char_conv(out): + """ + Convert integer vectors to character vectors for batch. + """ + out_conv = list() + for i in range(out.shape[0]): + tmp_str = '' + for j in range(out.shape[1]): + if int(out[i][j]) >= 0: + tmp_char = int2char(int(out[i][j])) + if int(out[i][j]) == 27: + tmp_char = '' + tmp_str = tmp_str + tmp_char + out_conv.append(tmp_str) + return out_conv diff --git a/example/gluon/lipnet/utils/download_data.py b/example/gluon/lipnet/utils/download_data.py new file mode 100644 index 000000000000..3051eb2a9e27 --- /dev/null +++ b/example/gluon/lipnet/utils/download_data.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: download_data +This module provides utilities for downloading the datasets for training LipNet +""" + +import os +from os.path import exists +from multi import multi_p_run, put_worker + + +def download_mp4(from_idx, to_idx, _params): + """ + download mp4s + """ + succ = set() + fail = set() + for idx in range(from_idx, to_idx): + name = 's' + str(idx) + save_folder = '{src_path}/{nm}'.format(src_path=_params['src_path'], nm=name) + if idx == 0 or os.path.isdir(save_folder): + continue + script = "http://spandh.dcs.shef.ac.uk/gridcorpus/{nm}/video/{nm}.mpg_vcd.zip".format( \ + nm=name) + down_sc = 'cd {src_path} && curl {script} --output {nm}.mpg_vcd.zip && \ + unzip {nm}.mpg_vcd.zip'.format(script=script, + nm=name, + src_path=_params['src_path']) + try: + print(down_sc) + os.system(down_sc) + succ.add(idx) + except OSError as error: + print(error) + fail.add(idx) + return (succ, fail) + + +def download_align(from_idx, to_idx, _params): + """ + download aligns + """ + succ = set() + fail = set() + for idx in range(from_idx, to_idx): + name = 's' + str(idx) + if idx == 0: + continue + script = "http://spandh.dcs.shef.ac.uk/gridcorpus/{nm}/align/{nm}.tar".format(nm=name) + down_sc = 'cd {align_path} && wget {script} && \ + tar -xvf {nm}.tar'.format(script=script, + nm=name, + align_path=_params['align_path']) + try: + print(down_sc) + os.system(down_sc) + succ.add(idx) + except OSError as error: + print(error) + fail.add(idx) + return (succ, fail) + + +if __name__ == '__main__': + import argparse + PARSER = argparse.ArgumentParser() + PARSER.add_argument('--src_path', type=str, default='../data/mp4s') + PARSER.add_argument('--align_path', type=str, default='../data') + PARSER.add_argument('--n_process', type=int, default=1) + CONFIG = PARSER.parse_args() + PARAMS = {'src_path': CONFIG.src_path, 'align_path': CONFIG.align_path} + N_PROCESS = CONFIG.n_process + + if exists('./shape_predictor_68_face_landmarks.dat') is False: + os.system('wget http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2 && \ + bzip2 -d shape_predictor_68_face_landmarks.dat.bz2') + + os.makedirs('{src_path}'.format(src_path=PARAMS['src_path']), exist_ok=True) + os.makedirs('{align_path}'.format(align_path=PARAMS['align_path']), exist_ok=True) + + if N_PROCESS == 1: + RES = download_mp4(0, 35, PARAMS) + RES = download_align(0, 35, PARAMS) + else: + # download movie files + RES = multi_p_run(tot_num=35, _func=put_worker, worker=download_mp4, \ + params=PARAMS, n_process=N_PROCESS) + + # download align files + RES = multi_p_run(tot_num=35, _func=put_worker, worker=download_align, \ + params=PARAMS, n_process=N_PROCESS) + + os.system('rm -f {src_path}/*.zip && rm -f {src_path}/*/Thumbs.db'.format( \ + src_path=PARAMS['src_path'])) + os.system('rm -f {align_path}/*.tar && rm -f {align_path}/Thumbs.db'.format( \ + align_path=PARAMS['align_path'])) diff --git a/example/gluon/lipnet/utils/multi.py b/example/gluon/lipnet/utils/multi.py new file mode 100644 index 000000000000..ce545b572de6 --- /dev/null +++ b/example/gluon/lipnet/utils/multi.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: preprocess with multi-process +""" + + +def multi_p_run(tot_num, _func, worker, params, n_process): + """ + Run _func with multi-process using params. + """ + from multiprocessing import Process, Queue + out_q = Queue() + procs = [] + + split_num = split_seq(list(range(0, tot_num)), n_process) + + print(tot_num, ">>", split_num) + + split_len = len(split_num) + if n_process > split_len: + n_process = split_len + + for i in range(n_process): + _p = Process(target=_func, + args=(worker, split_num[i][0], split_num[i][1], + params, out_q)) + _p.daemon = True + procs.append(_p) + _p.start() + + try: + result = [] + for i in range(n_process): + result.append(out_q.get()) + for i in procs: + i.join() + except KeyboardInterrupt: + print('Killing all the children in the pool.') + for i in procs: + i.terminate() + i.join() + return -1 + + while not out_q.empty(): + print(out_q.get(block=False)) + + return result + + +def split_seq(sam_num, n_tile): + """ + Split the number(sam_num) into numbers by n_tile + """ + import math + print(sam_num) + print(n_tile) + start_num = sam_num[0::int(math.ceil(len(sam_num) / (n_tile)))] + end_num = start_num[1::] + end_num.append(len(sam_num)) + return [[i, j] for i, j in zip(start_num, end_num)] + + +def put_worker(func, from_idx, to_idx, params, out_q): + """ + put worker + """ + succ, fail = func(from_idx, to_idx, params) + return out_q.put({'succ': succ, 'fail': fail}) + + +def test_worker(from_idx, to_idx, params): + """ + the worker to test multi-process + """ + params = params + succ = set() + fail = set() + for idx in range(from_idx, to_idx): + try: + succ.add(idx) + except ValueError: + fail.add(idx) + return (succ, fail) + + +if __name__ == '__main__': + RES = multi_p_run(35, put_worker, test_worker, params={}, n_process=5) + print(RES) diff --git a/example/gluon/lipnet/utils/preprocess_data.py b/example/gluon/lipnet/utils/preprocess_data.py new file mode 100644 index 000000000000..a13fad88af7a --- /dev/null +++ b/example/gluon/lipnet/utils/preprocess_data.py @@ -0,0 +1,262 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Module: preprocess_data +Reference: https://github.com/rizkiarm/LipNet +""" + +# pylint: disable=too-many-locals, no-self-use, c-extension-no-member + +import os +import fnmatch +import errno +import numpy as np +from scipy import ndimage +from scipy.misc import imresize +from skimage import io +import skvideo.io +import dlib + +def mkdir_p(path): + """ + Make a directory + """ + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def find_files(directory, pattern): + """ + Find files + """ + for root, _, files in os.walk(directory): + for basename in files: + if fnmatch.fnmatch(basename, pattern): + filename = os.path.join(root, basename) + yield filename + +class Video(object): + """ + Preprocess for Video + """ + def __init__(self, vtype='mouth', face_predictor_path=None): + if vtype == 'face' and face_predictor_path is None: + raise AttributeError('Face video need to be accompanied with face predictor') + self.face_predictor_path = face_predictor_path + self.vtype = vtype + self.face = None + self.mouth = None + self.data = None + self.length = None + + def from_frames(self, path): + """ + Read from frames + """ + frames_path = sorted([os.path.join(path, x) for x in os.listdir(path)]) + frames = [ndimage.imread(frame_path) for frame_path in frames_path] + self.handle_type(frames) + return self + + def from_video(self, path): + """ + Read from videos + """ + frames = self.get_video_frames(path) + self.handle_type(frames) + return self + + def from_array(self, frames): + """ + Read from array + """ + self.handle_type(frames) + return self + + def handle_type(self, frames): + """ + Config video types + """ + if self.vtype == 'mouth': + self.process_frames_mouth(frames) + elif self.vtype == 'face': + self.process_frames_face(frames) + else: + raise Exception('Video type not found') + + def process_frames_face(self, frames): + """ + Preprocess from frames using face detector + """ + detector = dlib.get_frontal_face_detector() + predictor = dlib.shape_predictor(self.face_predictor_path) + mouth_frames = self.get_frames_mouth(detector, predictor, frames) + self.face = np.array(frames) + self.mouth = np.array(mouth_frames) + if mouth_frames[0] is not None: + self.set_data(mouth_frames) + + def process_frames_mouth(self, frames): + """ + Preprocess from frames using mouth detector + """ + self.face = np.array(frames) + self.mouth = np.array(frames) + self.set_data(frames) + + def get_frames_mouth(self, detector, predictor, frames): + """ + Get frames using mouth crop + """ + mouth_width = 100 + mouth_height = 50 + horizontal_pad = 0.19 + normalize_ratio = None + mouth_frames = [] + for frame in frames: + dets = detector(frame, 1) + shape = None + for det in dets: + shape = predictor(frame, det) + i = -1 + if shape is None: # Detector doesn't detect face, just return None + return [None] + mouth_points = [] + for part in shape.parts(): + i += 1 + if i < 48: # Only take mouth region + continue + mouth_points.append((part.x, part.y)) + np_mouth_points = np.array(mouth_points) + + mouth_centroid = np.mean(np_mouth_points[:, -2:], axis=0) + + if normalize_ratio is None: + mouth_left = np.min(np_mouth_points[:, :-1]) * (1.0 - horizontal_pad) + mouth_right = np.max(np_mouth_points[:, :-1]) * (1.0 + horizontal_pad) + + normalize_ratio = mouth_width / float(mouth_right - mouth_left) + + new_img_shape = (int(frame.shape[0] * normalize_ratio), + int(frame.shape[1] * normalize_ratio)) + resized_img = imresize(frame, new_img_shape) + + mouth_centroid_norm = mouth_centroid * normalize_ratio + + mouth_l = int(mouth_centroid_norm[0] - mouth_width / 2) + mouth_r = int(mouth_centroid_norm[0] + mouth_width / 2) + mouth_t = int(mouth_centroid_norm[1] - mouth_height / 2) + mouth_b = int(mouth_centroid_norm[1] + mouth_height / 2) + + mouth_crop_image = resized_img[mouth_t:mouth_b, mouth_l:mouth_r] + + mouth_frames.append(mouth_crop_image) + return mouth_frames + + def get_video_frames(self, path): + """ + Get video frames + """ + videogen = skvideo.io.vreader(path) + frames = np.array([frame for frame in videogen]) + return frames + + def set_data(self, frames): + """ + Prepare the input of model + """ + data_frames = [] + for frame in frames: + #frame H x W x C + frame = frame.swapaxes(0, 1) # swap width and height to form format W x H x C + if len(frame.shape) < 3: + frame = np.array([frame]).swapaxes(0, 2).swapaxes(0, 1) # Add grayscale channel + data_frames.append(frame) + frames_n = len(data_frames) + data_frames = np.array(data_frames) # T x W x H x C + data_frames = np.rollaxis(data_frames, 3) # C x T x W x H + data_frames = data_frames.swapaxes(2, 3) # C x T x H x W = NCDHW + + self.data = data_frames + self.length = frames_n + +def preprocess(from_idx, to_idx, _params): + """ + Preprocess: Convert a video into the mouth images + """ + source_exts = '*.mpg' + src_path = _params['src_path'] + tgt_path = _params['tgt_path'] + face_predictor_path = './shape_predictor_68_face_landmarks.dat' + + succ = set() + fail = set() + for idx in range(from_idx, to_idx): + s_id = 's' + str(idx) + '/' + source_path = src_path + '/' + s_id + target_path = tgt_path + '/' + s_id + fail_cnt = 0 + for filepath in find_files(source_path, source_exts): + print("Processing: {}".format(filepath)) + filepath_wo_ext = os.path.splitext(filepath)[0].split('/')[-2:] + target_dir = os.path.join(tgt_path, '/'.join(filepath_wo_ext)) + + if os.path.exists(target_dir): + continue + + try: + video = Video(vtype='face', \ + face_predictor_path=face_predictor_path).from_video(filepath) + mkdir_p(target_dir) + i = 0 + if video.mouth[0] is None: + continue + for frame in video.mouth: + io.imsave(os.path.join(target_dir, "mouth_{0:03d}.png".format(i)), frame) + i += 1 + except ValueError as error: + print(error) + fail_cnt += 1 + if fail_cnt == 0: + succ.add(idx) + else: + fail.add(idx) + return (succ, fail) + +if __name__ == '__main__': + import argparse + from multi import multi_p_run, put_worker + PARSER = argparse.ArgumentParser() + PARSER.add_argument('--src_path', type=str, default='../data/mp4s') + PARSER.add_argument('--tgt_path', type=str, default='../data/datasets') + PARSER.add_argument('--n_process', type=int, default=1) + CONFIG = PARSER.parse_args() + N_PROCESS = CONFIG.n_process + PARAMS = {'src_path':CONFIG.src_path, + 'tgt_path':CONFIG.tgt_path} + + os.makedirs('{tgt_path}'.format(tgt_path=PARAMS['tgt_path']), exist_ok=True) + + if N_PROCESS == 1: + RES = preprocess(0, 35, PARAMS) + else: + RES = multi_p_run(35, put_worker, preprocess, PARAMS, N_PROCESS) diff --git a/example/gluon/lipnet/utils/run_preprocess.ipynb b/example/gluon/lipnet/utils/run_preprocess.ipynb new file mode 100644 index 000000000000..7a25e9b33517 --- /dev/null +++ b/example/gluon/lipnet/utils/run_preprocess.ipynb @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from download_data import multi_p_run, put_worker, _worker, download_mp4, download_align" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TEST" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]\n", + "5\n", + "35 >> [[0, 7], [7, 14], [14, 21], [21, 28], [28, 35]]\n", + "[{'succ': {0, 1, 2, 3, 4, 5, 6}, 'fail': set()}, {'succ': {7, 8, 9, 10, 11, 12, 13}, 'fail': set()}, {'succ': {14, 15, 16, 17, 18, 19, 20}, 'fail': set()}, {'succ': {21, 22, 23, 24, 25, 26, 27}, 'fail': set()}, {'succ': {32, 33, 34, 28, 29, 30, 31}, 'fail': set()}]\n" + ] + } + ], + "source": [ + "res = multi_p_run(35, put_worker, _worker, 5)\n", + "print (res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## down\n", + "import os\n", + "os.makedirs('./datasets', exist_ok=True)\n", + "#os.system('rm -rf ./datasets/*')\n", + "\n", + "res = multi_p_run(35, put_worker, download_align, 9)\n", + "print (res)\n", + "\n", + "os.system('rm -f datasets/*.tar && rm -f datasets/align/Thumbs.db')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "res = multi_p_run(35, put_worker, download_mp4, 9)\n", + "print (res)\n", + "\n", + "os.system('rm -f datasets/*.zip && rm -f datasets/*/Thumbs.db')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## download single 22 th dir\n", + "#download_data.py(22, 22)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preprocess Data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from preprocess_data import preprocess, find_files, Video" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "os.makedirs('./TARGET', exist_ok=True)\n", + "os.system('rm -rf ./TARGET/*')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]\n", + "9\n", + "35 >> [[0, 4], [4, 8], [8, 12], [12, 16], [16, 20], [20, 24], [24, 28], [28, 32], [32, 35]]\n", + "Processing: datasets/s1/prwq3s.mpg\n", + "Processing: datasets/s4/lrix7n.mpg\n", + "Processing: datasets/s8/pgbyza.mpg\n", + "Processing: datasets/s12/brik7n.mpg\n", + "Processing: datasets/s16/sgit7p.mpg\n", + "Processing: datasets/s20/lrbp8a.mpg\n", + "Processing: datasets/s24/sbik8a.mpg\n", + "Processing: datasets/s28/srwf8a.mpg\n", + "Processing: datasets/s32/pbbm1n.mpg\n", + "Processing: datasets/s12/sbbaza.mpg\n", + "Processing: datasets/s28/lbit7n.mpg\n", + "Processing: datasets/s32/pbwm7p.mpg\n", + "Processing: datasets/s8/bril2s.mpg\n", + "Processing: datasets/s20/bway7n.mpg\n", + "Processing: datasets/s1/pbib8p.mpg\n", + "Processing: datasets/s16/lwaj7n.mpg\n", + "Processing: datasets/s24/bwwl6a.mpg\n", + "Processing: datasets/s4/bbwf7n.mpg\n" + ] + } + ], + "source": [ + "res = multi_p_run(35, put_worker, preprocess, 9)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/example/gluon/lipnet/utils/run_preprocess_single_process.ipynb b/example/gluon/lipnet/utils/run_preprocess_single_process.ipynb new file mode 100644 index 000000000000..4311323206e1 --- /dev/null +++ b/example/gluon/lipnet/utils/run_preprocess_single_process.ipynb @@ -0,0 +1,360 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from download_data import multi_p_run, put_worker, test_worker, download_mp4, download_align" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "tot_movies=35" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TEST" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]\n", + "5\n", + "35 >> [[0, 7], [7, 14], [14, 21], [21, 28], [28, 35]]\n", + "[{'succ': {0, 1, 2, 3, 4, 5, 6}, 'fail': set()}, {'succ': {7, 8, 9, 10, 11, 12, 13}, 'fail': set()}, {'succ': {14, 15, 16, 17, 18, 19, 20}, 'fail': set()}, {'succ': {21, 22, 23, 24, 25, 26, 27}, 'fail': set()}, {'succ': {32, 33, 34, 28, 29, 30, 31}, 'fail': set()}]\n" + ] + } + ], + "source": [ + "res = multi_p_run(tot_movies, put_worker, test_worker, params={}, n_process=5)\n", + "print (res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Aligns" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s0/align/s0.tar && tar -xvf s0.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s1/align/s1.tar && tar -xvf s1.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s2/align/s2.tar && tar -xvf s2.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s3/align/s3.tar && tar -xvf s3.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s4/align/s4.tar && tar -xvf s4.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s5/align/s5.tar && tar -xvf s5.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s6/align/s6.tar && tar -xvf s6.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s7/align/s7.tar && tar -xvf s7.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s8/align/s8.tar && tar -xvf s8.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s9/align/s9.tar && tar -xvf s9.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s10/align/s10.tar && tar -xvf s10.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s11/align/s11.tar && tar -xvf s11.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s12/align/s12.tar && tar -xvf s12.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s13/align/s13.tar && tar -xvf s13.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s14/align/s14.tar && tar -xvf s14.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s15/align/s15.tar && tar -xvf s15.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s16/align/s16.tar && tar -xvf s16.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s17/align/s17.tar && tar -xvf s17.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s18/align/s18.tar && tar -xvf s18.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s19/align/s19.tar && tar -xvf s19.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s20/align/s20.tar && tar -xvf s20.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s21/align/s21.tar && tar -xvf s21.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s22/align/s22.tar && tar -xvf s22.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s23/align/s23.tar && tar -xvf s23.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s24/align/s24.tar && tar -xvf s24.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s25/align/s25.tar && tar -xvf s25.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s26/align/s26.tar && tar -xvf s26.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s27/align/s27.tar && tar -xvf s27.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s28/align/s28.tar && tar -xvf s28.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s29/align/s29.tar && tar -xvf s29.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s30/align/s30.tar && tar -xvf s30.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s31/align/s31.tar && tar -xvf s31.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s32/align/s32.tar && tar -xvf s32.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s33/align/s33.tar && tar -xvf s33.tar\n", + "cd ../data/align && wget http://spandh.dcs.shef.ac.uk/gridcorpus/s34/align/s34.tar && tar -xvf s34.tar\n" + ] + } + ], + "source": [ + "align_path = '../data/align'\n", + "os.makedirs(align_path, exist_ok=True)\n", + "\n", + "res = download_align(0, tot_movies, {'align_path':align_path})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34}, set())\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print (res)\n", + "os.system('rm -f {align_path}/*.tar && rm -f {align_path}/Thumbs.db'.format(align_path=align_path))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "### Moives(MP4s)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s0/video/s0.mpg_vcd.zip --output s0.mpg_vcd.zip && unzip s0.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s1/video/s1.mpg_vcd.zip --output s1.mpg_vcd.zip && unzip s1.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s2/video/s2.mpg_vcd.zip --output s2.mpg_vcd.zip && unzip s2.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s3/video/s3.mpg_vcd.zip --output s3.mpg_vcd.zip && unzip s3.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s4/video/s4.mpg_vcd.zip --output s4.mpg_vcd.zip && unzip s4.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s5/video/s5.mpg_vcd.zip --output s5.mpg_vcd.zip && unzip s5.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s6/video/s6.mpg_vcd.zip --output s6.mpg_vcd.zip && unzip s6.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s7/video/s7.mpg_vcd.zip --output s7.mpg_vcd.zip && unzip s7.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s8/video/s8.mpg_vcd.zip --output s8.mpg_vcd.zip && unzip s8.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s9/video/s9.mpg_vcd.zip --output s9.mpg_vcd.zip && unzip s9.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s10/video/s10.mpg_vcd.zip --output s10.mpg_vcd.zip && unzip s10.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s11/video/s11.mpg_vcd.zip --output s11.mpg_vcd.zip && unzip s11.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s12/video/s12.mpg_vcd.zip --output s12.mpg_vcd.zip && unzip s12.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s13/video/s13.mpg_vcd.zip --output s13.mpg_vcd.zip && unzip s13.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s14/video/s14.mpg_vcd.zip --output s14.mpg_vcd.zip && unzip s14.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s15/video/s15.mpg_vcd.zip --output s15.mpg_vcd.zip && unzip s15.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s16/video/s16.mpg_vcd.zip --output s16.mpg_vcd.zip && unzip s16.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s17/video/s17.mpg_vcd.zip --output s17.mpg_vcd.zip && unzip s17.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s18/video/s18.mpg_vcd.zip --output s18.mpg_vcd.zip && unzip s18.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s19/video/s19.mpg_vcd.zip --output s19.mpg_vcd.zip && unzip s19.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s20/video/s20.mpg_vcd.zip --output s20.mpg_vcd.zip && unzip s20.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s21/video/s21.mpg_vcd.zip --output s21.mpg_vcd.zip && unzip s21.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s22/video/s22.mpg_vcd.zip --output s22.mpg_vcd.zip && unzip s22.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s23/video/s23.mpg_vcd.zip --output s23.mpg_vcd.zip && unzip s23.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s24/video/s24.mpg_vcd.zip --output s24.mpg_vcd.zip && unzip s24.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s25/video/s25.mpg_vcd.zip --output s25.mpg_vcd.zip && unzip s25.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s26/video/s26.mpg_vcd.zip --output s26.mpg_vcd.zip && unzip s26.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s27/video/s27.mpg_vcd.zip --output s27.mpg_vcd.zip && unzip s27.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s28/video/s28.mpg_vcd.zip --output s28.mpg_vcd.zip && unzip s28.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s29/video/s29.mpg_vcd.zip --output s29.mpg_vcd.zip && unzip s29.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s30/video/s30.mpg_vcd.zip --output s30.mpg_vcd.zip && unzip s30.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s31/video/s31.mpg_vcd.zip --output s31.mpg_vcd.zip && unzip s31.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s32/video/s32.mpg_vcd.zip --output s32.mpg_vcd.zip && unzip s32.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s33/video/s33.mpg_vcd.zip --output s33.mpg_vcd.zip && unzip s33.mpg_vcd.zip\n", + "cd ../data/mp4s && curl http://spandh.dcs.shef.ac.uk/gridcorpus/s34/video/s34.mpg_vcd.zip --output s34.mpg_vcd.zip && unzip s34.mpg_vcd.zip\n" + ] + } + ], + "source": [ + "src_path = '../data/mp4s'\n", + "res = download_mp4(0, tot_movies, {'src_path':src_path})" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34}, set())\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print (res)\n", + "os.system('rm -f {src_path}/*.zip && rm -f {src_path}/*/Thumbs.db'.format(src_path=src_path))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preprocess Data" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from preprocess_data import preprocess, find_files, Video" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "tgt_path = '../data/datasets'" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.makedirs('{tgt_path}'.format(tgt_path=tgt_path), exist_ok=True)\n", + "os.system('rm -rf {tgt_path}'.format(tgt_path=tgt_path))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "res = preprocess(0, tot_movies, {'src_path':src_path, 'tgt_path':tgt_path})" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34}, set())\n" + ] + } + ], + "source": [ + "print (res)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [default]", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/example/gluon/lstm_crf/README.md b/example/gluon/lstm_crf/README.md new file mode 100644 index 000000000000..519c3b89f9fd --- /dev/null +++ b/example/gluon/lstm_crf/README.md @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + +# BiLSTM CRF model +This example demonstrates how a [BiLSTM-CRF model](https://arxiv.org/pdf/1508.01991v1.pdf) can be implemented in Gluon to perform noun-phrase chunking as a sequence labeling task. In this example we define the following training sample: +``` +georgia tech is a university in georgia +B I O O O O B +``` +The second line is the IOB representation of the above sentence that is learnt by the model. **I** stands for in chunk, **O** for out of a chunk and **B** for beginning of junks. + +The model consists of an LSTM layer with 2 hidden units and a CRF layer. The CRF layer has a state transition matrix which allows to take past and future tags into account when predicting the current tag. The bidirectional LSTM is reading the word sequence from beginning to end and vice versa. It prodcues a vector representation for the words. The following image is taken from https://arxiv.org/pdf/1508.01991v1.pdf and shows the model architecture: + +![Image taken from https://arxiv.org/pdf/1508.01991v1.pdf](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gluon/lstm_crf/bi-lstm_crf.png) + +You can run the example by executing +``` +python lstm_crf.py +``` +The example code does not take any commandline arguments. If you want to change the number of hidden units or the size of vectors embeddings, then you need to change the variables ```EMBEDDING_DIM``` and ```HIDDEN_DIM```. + + diff --git a/example/gluon/mnist/README.md b/example/gluon/mnist/README.md new file mode 100644 index 000000000000..c053364fad3c --- /dev/null +++ b/example/gluon/mnist/README.md @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + +# MNIST classification example + +This script shows a simple example how to do image classification with Gluon. +The model is trained on MNIST digits image dataset and the goal is to classify the digits ```0-9```. The model has the following layout: +``` +net = nn.Sequential() +with net.name_scope(): + net.add(nn.Dense(128, activation='relu')) + net.add(nn.Dense(64, activation='relu')) + net.add(nn.Dense(10)) +``` + +The script provides the following commandline arguments: + + +``` +MXNet Gluon MNIST Example + +optional arguments: + -h, --help show this help message and exit + --batch-size BATCH_SIZE + batch size for training and testing (default: 100) + --epochs EPOCHS number of epochs to train (default: 10) + --lr LR learning rate (default: 0.1) + --momentum MOMENTUM SGD momentum (default: 0.9) + --cuda Train on GPU with CUDA + --log-interval N how many batches to wait before logging training + status +``` + +After one epoch we get the following output vector for the given test image: + + + +[-5.461655 -4.745 -1.8203478 -0.5705207 8.923972 -2.2358544 -3.3020825 -2.409004 4.0074944 10.362008] + +As we can see the highest activation is 10.362 which corresponds to label `9`. + diff --git a/example/gluon/sn_gan/README.md b/example/gluon/sn_gan/README.md index 5b2a750e4efb..054416fced09 100644 --- a/example/gluon/sn_gan/README.md +++ b/example/gluon/sn_gan/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Spectral Normalization GAN This example implements [Spectral Normalization for Generative Adversarial Networks](https://arxiv.org/abs/1802.05957) based on [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) dataset. diff --git a/example/gluon/style_transfer/README.md b/example/gluon/style_transfer/README.md index ef273a5975ab..c182f3f8fce8 100644 --- a/example/gluon/style_transfer/README.md +++ b/example/gluon/style_transfer/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet-Gluon-Style-Transfer This repo provides MXNet Implementation of **[Neural Style Transfer](#neural-style)** and **[MSG-Net](#real-time-style-transfer)**. diff --git a/example/gluon/super_resolution/README.md b/example/gluon/super_resolution/README.md new file mode 100644 index 000000000000..ddcbe8b0a202 --- /dev/null +++ b/example/gluon/super_resolution/README.md @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + +# Superresolution + +This example trains a convolutional neural network to enhance the resolution of images (also known as superresolution). +The script takes the following commandline arguments: + +``` +Super-resolution using an efficient sub-pixel convolution neural network. + +optional arguments: + -h, --help show this help message and exit + --upscale_factor UPSCALE_FACTOR + super resolution upscale factor. default is 3. + --batch_size BATCH_SIZE + training batch size, per device. default is 4. + --test_batch_size TEST_BATCH_SIZE + test batch size + --epochs EPOCHS number of training epochs + --lr LR learning Rate. default is 0.001. + --use-gpu whether to use GPU. + --seed SEED random seed to use. Default=123 + --resolve_img RESOLVE_IMG + input image to use +``` + +Once the network is trained you can use the following command to increase the resolution of your image: +``` +python super_resolution.py --resolve_img myimage.jpg +``` diff --git a/example/gluon/data.py b/example/gluon/super_resolution/data.py similarity index 100% rename from example/gluon/data.py rename to example/gluon/super_resolution/data.py diff --git a/example/gluon/super_resolution/super_resolution.py b/example/gluon/super_resolution/super_resolution.py index 0f2f21f3c0a7..198f6fe0611b 100644 --- a/example/gluon/super_resolution/super_resolution.py +++ b/example/gluon/super_resolution/super_resolution.py @@ -16,19 +16,27 @@ # under the License. from __future__ import print_function -import argparse, tarfile + +import argparse import math import os +import shutil +import sys +import zipfile +from os import path + import numpy as np import mxnet as mx -import mxnet.ndarray as F -from mxnet import gluon +from mxnet import gluon, autograd as ag from mxnet.gluon import nn -from mxnet import autograd as ag -from mxnet.test_utils import download +from mxnet.gluon.contrib import nn as contrib_nn from mxnet.image import CenterCropAug, ResizeAug from mxnet.io import PrefetchingIter +from mxnet.test_utils import download + +this_dir = path.abspath(path.dirname(__file__)) +sys.path.append(path.join(this_dir, path.pardir)) from data import ImagePairIter @@ -51,19 +59,45 @@ batch_size, test_batch_size = opt.batch_size, opt.test_batch_size color_flag = 0 -# get data -dataset_path = "dataset" -dataset_url = "http://www2.eecs.berkeley.edu/Research/Projects/CS/vision/bsds/BSDS300-images.tgz" -def get_dataset(prefetch=False): - image_path = os.path.join(dataset_path, "BSDS300/images") +# Get data +datasets_dir = path.expanduser(path.join("~", ".mxnet", "datasets")) +datasets_tmpdir = path.join(datasets_dir, "tmp") +dataset_url = "https://github.com/BIDS/BSDS500/archive/master.zip" +data_dir = path.expanduser(path.join(datasets_dir, "BSDS500")) +tmp_dir = path.join(data_dir, "tmp") - if not os.path.exists(image_path): - os.makedirs(dataset_path) - file_name = download(dataset_url) - with tarfile.open(file_name) as tar: - for item in tar: - tar.extract(item, dataset_path) - os.remove(file_name) +def get_dataset(prefetch=False): + """Download the BSDS500 dataset and return train and test iters.""" + + if path.exists(data_dir): + print( + "Directory {} already exists, skipping.\n" + "To force download and extraction, delete the directory and re-run." + "".format(data_dir), + file=sys.stderr, + ) + else: + print("Downloading dataset...", file=sys.stderr) + downloaded_file = download(dataset_url, dirname=datasets_tmpdir) + print("done", file=sys.stderr) + + print("Extracting files...", end="", file=sys.stderr) + os.makedirs(data_dir) + os.makedirs(tmp_dir) + with zipfile.ZipFile(downloaded_file) as archive: + archive.extractall(tmp_dir) + shutil.rmtree(datasets_tmpdir) + + shutil.copytree( + path.join(tmp_dir, "BSDS500-master", "BSDS500", "data", "images"), + path.join(data_dir, "images"), + ) + shutil.copytree( + path.join(tmp_dir, "BSDS500-master", "BSDS500", "data", "groundTruth"), + path.join(data_dir, "groundTruth"), + ) + shutil.rmtree(tmp_dir) + print("done", file=sys.stderr) crop_size = 256 crop_size -= crop_size % upscale_factor @@ -72,15 +106,26 @@ def get_dataset(prefetch=False): input_transform = [CenterCropAug((crop_size, crop_size)), ResizeAug(input_crop_size)] target_transform = [CenterCropAug((crop_size, crop_size))] - iters = (ImagePairIter(os.path.join(image_path, "train"), - (input_crop_size, input_crop_size), - (crop_size, crop_size), - batch_size, color_flag, input_transform, target_transform), - ImagePairIter(os.path.join(image_path, "test"), - (input_crop_size, input_crop_size), - (crop_size, crop_size), - test_batch_size, color_flag, - input_transform, target_transform)) + iters = ( + ImagePairIter( + path.join(data_dir, "images", "train"), + (input_crop_size, input_crop_size), + (crop_size, crop_size), + batch_size, + color_flag, + input_transform, + target_transform, + ), + ImagePairIter( + path.join(data_dir, "images", "test"), + (input_crop_size, input_crop_size), + (crop_size, crop_size), + test_batch_size, + color_flag, + input_transform, + target_transform, + ), + ) return [PrefetchingIter(i) for i in iters] if prefetch else iters @@ -90,33 +135,23 @@ def get_dataset(prefetch=False): ctx = [mx.gpu(0)] if opt.use_gpu else [mx.cpu()] -# define model -def _rearrange(raw, F, upscale_factor): - # (N, C * r^2, H, W) -> (N, C, r^2, H, W) - splitted = F.reshape(raw, shape=(0, -4, -1, upscale_factor**2, 0, 0)) - # (N, C, r^2, H, W) -> (N, C, r, r, H, W) - unflatten = F.reshape(splitted, shape=(0, 0, -4, upscale_factor, upscale_factor, 0, 0)) - # (N, C, r, r, H, W) -> (N, C, H, r, W, r) - swapped = F.transpose(unflatten, axes=(0, 1, 4, 2, 5, 3)) - # (N, C, H, r, W, r) -> (N, C, H*r, W*r) - return F.reshape(swapped, shape=(0, 0, -3, -3)) - - -class SuperResolutionNet(gluon.Block): +class SuperResolutionNet(gluon.HybridBlock): def __init__(self, upscale_factor): super(SuperResolutionNet, self).__init__() with self.name_scope(): - self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2)) - self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1)) - self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1)) + self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2), activation='relu') + self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1), activation='relu') + self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1), activation='relu') self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1)) - self.upscale_factor = upscale_factor + self.pxshuf = contrib_nn.PixelShuffle2D(upscale_factor) - def forward(self, x): - x = F.Activation(self.conv1(x), act_type='relu') - x = F.Activation(self.conv2(x), act_type='relu') - x = F.Activation(self.conv3(x), act_type='relu') - return _rearrange(self.conv4(x), F, self.upscale_factor) + def hybrid_forward(self, F, x): + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + x = self.conv4(x) + x = self.pxshuf(x) + return x net = SuperResolutionNet(upscale_factor) metric = mx.metric.MSE() @@ -136,7 +171,7 @@ def test(ctx): avg_psnr += 10 * math.log10(1/metric.get()[1]) metric.reset() avg_psnr /= batches - print('validation avg psnr: %f'%avg_psnr) + print('validation avg psnr: %f' % avg_psnr) def train(epoch, ctx): @@ -168,13 +203,18 @@ def train(epoch, ctx): print('training mse at epoch %d: %s=%f'%(i, name, acc)) test(ctx) - net.save_parameters('superres.params') + net.save_parameters(path.join(this_dir, 'superres.params')) def resolve(ctx): from PIL import Image + if isinstance(ctx, list): ctx = [ctx[0]] - net.load_parameters('superres.params', ctx=ctx) + + img_basename = path.splitext(path.basename(opt.resolve_img))[0] + img_dirname = path.dirname(opt.resolve_img) + + net.load_parameters(path.join(this_dir, 'superres.params'), ctx=ctx) img = Image.open(opt.resolve_img).convert('YCbCr') y, cb, cr = img.split() data = mx.nd.expand_dims(mx.nd.expand_dims(mx.nd.array(y), axis=0), axis=0) @@ -186,7 +226,7 @@ def resolve(ctx): out_img_cr = cr.resize(out_img_y.size, Image.BICUBIC) out_img = Image.merge('YCbCr', [out_img_y, out_img_cb, out_img_cr]).convert('RGB') - out_img.save('resolved.png') + out_img.save(path.join(img_dirname, '{}-resolved.png'.format(img_basename))) if opt.resolve_img: resolve(ctx) diff --git a/example/gluon/tree_lstm/README.md b/example/gluon/tree_lstm/README.md index e14ab4c70afc..8e3b385b77b0 100644 --- a/example/gluon/tree_lstm/README.md +++ b/example/gluon/tree_lstm/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Tree-Structured Long Short-Term Memory Networks This is a [MXNet Gluon](https://mxnet.io/) implementation of Tree-LSTM as described in the paper [Improved Semantic Representations From Tree-Structured Long Short-Term Memory Networks](http://arxiv.org/abs/1503.00075) by Kai Sheng Tai, Richard Socher, and Christopher Manning. diff --git a/example/gluon/word_language_model/README.md b/example/gluon/word_language_model/README.md index 4a77950d01bc..04a4a4ac70b8 100644 --- a/example/gluon/word_language_model/README.md +++ b/example/gluon/word_language_model/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Word-level language modeling RNN This example trains a multi-layer RNN (Elman, GRU, or LSTM) on WikiText-2 language modeling benchmark. diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc index 2a605b8b2674..3c72589399ba 100644 --- a/example/image-classification/predict-cpp/image-classification-predict.cc +++ b/example/image-classification/predict-cpp/image-classification-predict.cc @@ -76,7 +76,9 @@ class BufferFile { ifs.seekg(0, std::ios::beg); std::cout << file_path.c_str() << " ... " << length_ << " bytes\n"; - buffer_.reset(new char[length_]); + // Buffer as null terminated to be converted to string + buffer_.reset(new char[length_ + 1]); + buffer_[length_] = 0; ifs.read(buffer_.get(), length_); ifs.close(); } diff --git a/example/kaggle-ndsb1/README.md b/example/kaggle-ndsb1/README.md index 82a99f7b6947..804ed9474dc9 100644 --- a/example/kaggle-ndsb1/README.md +++ b/example/kaggle-ndsb1/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Tutorial for Kaggle NDSB-1 ----- diff --git a/example/kaggle-ndsb2/README.md b/example/kaggle-ndsb2/README.md index 302e54033c8a..62a751b69302 100644 --- a/example/kaggle-ndsb2/README.md +++ b/example/kaggle-ndsb2/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # End-to-End Deep Learning Tutorial for Kaggle NDSB-II In this example, we will demo how to use MXNet to build an end-to-end deep learning system to help Diagnose Heart Disease. The demo network is able to achieve 0.039222 CRPS on validation set, which is good enough to get Top-10 (on Dec 22nd, 2015). diff --git a/example/model-parallel/README.md b/example/model-parallel/README.md new file mode 100644 index 000000000000..537562070a62 --- /dev/null +++ b/example/model-parallel/README.md @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + +# Run parts of a model on different devices + +This folder contains the example [matrix_factorization](https://github.com/apache/incubator-mxnet/tree/master/example/model-parallel/matrix_factorization) that demonstrates the basic usage of `group2ctxs`. diff --git a/example/model-parallel/matrix_factorization/README.md b/example/model-parallel/matrix_factorization/README.md index 00507d924f81..94ec91d0e8b8 100644 --- a/example/model-parallel/matrix_factorization/README.md +++ b/example/model-parallel/matrix_factorization/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Model Parallel Matrix Factorization =================================== diff --git a/example/module/README.md b/example/module/README.md index 99dd756ead63..24c917d5c8bc 100644 --- a/example/module/README.md +++ b/example/module/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Module Usage Example This folder contains usage examples for MXNet module. diff --git a/example/multi-task/README.md b/example/multi-task/README.md index b7756fe378a7..a110fd0bd7cf 100644 --- a/example/multi-task/README.md +++ b/example/multi-task/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Mulit-task learning example This is a simple example to show how to use mxnet for multi-task learning. It uses MNIST as an example, trying to predict jointly the digit and whether this digit is odd or even. diff --git a/example/multivariate_time_series/README.md b/example/multivariate_time_series/README.md index 87baca36d35f..c3db6c130759 100644 --- a/example/multivariate_time_series/README.md +++ b/example/multivariate_time_series/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # LSTNet - This repo contains an MXNet implementation of [this](https://arxiv.org/pdf/1703.07015.pdf) state of the art time series forecasting model. diff --git a/example/named_entity_recognition/README.md b/example/named_entity_recognition/README.md index c914a6985dfe..eaa358d4f18c 100644 --- a/example/named_entity_recognition/README.md +++ b/example/named_entity_recognition/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## Goal - This repo contains an MXNet implementation of this state of the art [entity recognition model](https://www.aclweb.org/anthology/Q16-1026). diff --git a/example/nce-loss/README.md b/example/nce-loss/README.md index 56e43525a7ca..26b028001f85 100644 --- a/example/nce-loss/README.md +++ b/example/nce-loss/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Examples of NCE Loss [Noise-contrastive estimation](http://proceedings.mlr.press/v9/gutmann10a/gutmann10a.pdf) loss (nce-loss) is used to speedup multi-class classification when class num is huge. diff --git a/example/neural-style/README.md b/example/neural-style/README.md index 5c4b58924827..e8e69ccc2194 100644 --- a/example/neural-style/README.md +++ b/example/neural-style/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Neural art This is an implementation of the paper diff --git a/example/neural-style/end_to_end/README.md b/example/neural-style/end_to_end/README.md index 4a228c199bb7..209d98df7254 100644 --- a/example/neural-style/end_to_end/README.md +++ b/example/neural-style/end_to_end/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # End to End Neural Art Please refer to this [blog](http://dmlc.ml/mxnet/2016/06/20/end-to-end-neural-style.html) for details of how it is implemented. diff --git a/example/numpy-ops/README.md b/example/numpy-ops/README.md index aa4911f67414..61bf8365bd18 100644 --- a/example/numpy-ops/README.md +++ b/example/numpy-ops/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Training with Custom Operators in Python These examples demonstrate custom operator implementations in python. diff --git a/example/profiler/README.md b/example/profiler/README.md index 1b9279ccf227..255ad8f65579 100644 --- a/example/profiler/README.md +++ b/example/profiler/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Profiler Examples This folder contains examples of using MXNet profiler to generate profiling results in json files. diff --git a/example/quantization/imagenet_inference.py b/example/quantization/imagenet_inference.py index 0725165b0ca5..47e206303e99 100644 --- a/example/quantization/imagenet_inference.py +++ b/example/quantization/imagenet_inference.py @@ -19,6 +19,7 @@ import logging import os import time +import numpy as np import mxnet as mx from mxnet import nd from mxnet.contrib.quantization import * @@ -98,7 +99,7 @@ def score(sym, arg_params, aux_params, data, devs, label_name, max_num_examples, logger.info(m.get()) -def benchmark_score(symbol_file, ctx, batch_size, num_batches, logger=None): +def benchmark_score(symbol_file, ctx, batch_size, num_batches, data_layer_type, logger=None): # get mod cur_path = os.path.dirname(os.path.realpath(__file__)) symbol_file_path = os.path.join(cur_path, symbol_file) @@ -106,14 +107,28 @@ def benchmark_score(symbol_file, ctx, batch_size, num_batches, logger=None): logger.info('Loading symbol from file %s' % symbol_file_path) sym = mx.sym.load(symbol_file_path) mod = mx.mod.Module(symbol=sym, context=ctx) - mod.bind(for_training = False, - inputs_need_grad = False, - data_shapes = [('data', (batch_size,)+data_shape)]) + if data_layer_type == "int8": + dshape = mx.io.DataDesc(name='data', shape=( + batch_size,) + data_shape, dtype=np.int8) + elif data_layer_type == 'uint8': + dshape = mx.io.DataDesc(name='data', shape=( + batch_size,) + data_shape, dtype=np.uint8) + else: # float32 + dshape = mx.io.DataDesc(name='data', shape=( + batch_size,) + data_shape, dtype=np.float32) + mod.bind(for_training=False, + inputs_need_grad=False, + data_shapes=[dshape]) mod.init_params(initializer=mx.init.Xavier(magnitude=2.)) # get data - data = [mx.random.uniform(-1.0, 1.0, shape=shape, ctx=ctx) for _, shape in mod.data_shapes] - batch = mx.io.DataBatch(data, []) # empty label + if data_layer_type == "float32": + data = [mx.random.uniform(-1.0, 1.0, shape=shape, ctx=ctx, dtype=data_layer_type) + for _, shape in mod.data_shapes] + else: + data = [mx.nd.full(shape=shape, val=127, ctx=ctx, dtype=data_layer_type) + for _, shape in mod.data_shapes] + batch = mx.io.DataBatch(data, []) # empty label # run dry_run = 5 # use 5 iterations to warm up @@ -152,6 +167,9 @@ def benchmark_score(symbol_file, ctx, batch_size, num_batches, logger=None): help='shuffling seed, see' ' https://mxnet.incubator.apache.org/api/python/io/io.html?highlight=imager#mxnet.io.ImageRecordIter' ' for more details') + parser.add_argument('--data-layer-type', type=str, default="float32", + choices=['float32', 'int8', 'uint8'], + help='data type for data layer') args = parser.parse_args() @@ -192,24 +210,52 @@ def benchmark_score(symbol_file, ctx, batch_size, num_batches, logger=None): data_shape = tuple([int(i) for i in image_shape.split(',')]) logger.info('Input data shape = %s' % str(data_shape)) + data_layer_type = args.data_layer_type if args.benchmark == False: dataset = args.dataset download_dataset('http://data.mxnet.io/data/val_256_q90.rec', dataset) logger.info('Dataset for inference: %s' % dataset) # creating data iterator - data = mx.io.ImageRecordIter(path_imgrec=dataset, - label_width=1, - preprocess_threads=data_nthreads, - batch_size=batch_size, - data_shape=data_shape, - label_name=label_name, - rand_crop=False, - rand_mirror=False, - shuffle=True, - shuffle_chunk_seed=3982304, - seed=48564309, - **combine_mean_std) + if data_layer_type == 'int8': + data = mx.io.ImageRecordInt8Iter(path_imgrec=dataset, + label_width=1, + preprocess_threads=data_nthreads, + batch_size=batch_size, + data_shape=data_shape, + label_name=label_name, + rand_crop=False, + rand_mirror=False, + shuffle=args.shuffle_dataset, + shuffle_chunk_seed=args.shuffle_chunk_seed, + seed=args.shuffle_seed, + **combine_mean_std) + elif data_layer_type == 'uint8': + data = mx.io.ImageRecordUInt8Iter(path_imgrec=dataset, + label_width=1, + preprocess_threads=data_nthreads, + batch_size=batch_size, + data_shape=data_shape, + label_name=label_name, + rand_crop=False, + rand_mirror=False, + shuffle=args.shuffle_dataset, + shuffle_chunk_seed=args.shuffle_chunk_seed, + seed=args.shuffle_seed, + **combine_mean_std) + else: #float32 + data = mx.io.ImageRecordIter(path_imgrec=dataset, + label_width=1, + preprocess_threads=data_nthreads, + batch_size=batch_size, + data_shape=data_shape, + label_name=label_name, + rand_crop=False, + rand_mirror=False, + shuffle=args.shuffle_dataset, + shuffle_chunk_seed=args.shuffle_chunk_seed, + seed=args.shuffle_seed, + **combine_mean_std) # loading model sym, arg_params, aux_params = load_model(symbol_file, param_file, logger) @@ -224,5 +270,5 @@ def benchmark_score(symbol_file, ctx, batch_size, num_batches, logger=None): max_num_examples=num_inference_images, logger=logger) else: logger.info('Running model %s for inference' % symbol_file) - speed = benchmark_score(symbol_file, ctx, batch_size, args.num_inference_batches, logger) + speed = benchmark_score(symbol_file, ctx, batch_size, args.num_inference_batches, data_layer_type, logger) logger.info('batch size %2d, image/sec: %f', batch_size, speed) diff --git a/example/rcnn/README.md b/example/rcnn/README.md index 5e6127ccb08d..e773089f7219 100644 --- a/example/rcnn/README.md +++ b/example/rcnn/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Faster R-CNN in MXNet Please redirect any issue or question of using this symbolic example of Faster R-CNN to https://github.com/ijkguo/mx-rcnn. diff --git a/example/recommenders/README.md b/example/recommenders/README.md index 628182c849b8..806e51f24c8d 100644 --- a/example/recommenders/README.md +++ b/example/recommenders/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Recommender Systems diff --git a/example/reinforcement-learning/a3c/README.md b/example/reinforcement-learning/a3c/README.md index 5eaba66a5b86..55ade6a279f4 100644 --- a/example/reinforcement-learning/a3c/README.md +++ b/example/reinforcement-learning/a3c/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # A3C Implementation This is an attempt to implement the A3C algorithm in paper Asynchronous Methods for Deep Reinforcement Learning. diff --git a/example/reinforcement-learning/ddpg/README.md b/example/reinforcement-learning/ddpg/README.md index 2e299dd5daa3..69e00d982f98 100644 --- a/example/reinforcement-learning/ddpg/README.md +++ b/example/reinforcement-learning/ddpg/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # mx-DDPG MXNet Implementation of DDPG diff --git a/example/reinforcement-learning/parallel_actor_critic/README.md b/example/reinforcement-learning/parallel_actor_critic/README.md index d3288492a611..5a442079b3a2 100644 --- a/example/reinforcement-learning/parallel_actor_critic/README.md +++ b/example/reinforcement-learning/parallel_actor_critic/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # 'Parallel Advantage-Actor Critic' Implementation This repo contains a MXNet implementation of a variant of the A3C algorithm from [Asynchronous Methods for Deep Reinforcement Learning](https://arxiv.org/pdf/1602.01783v2.pdf). diff --git a/example/restricted-boltzmann-machine/README.md b/example/restricted-boltzmann-machine/README.md index a8769a51e05a..d1609049c137 100644 --- a/example/restricted-boltzmann-machine/README.md +++ b/example/restricted-boltzmann-machine/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Restricted Boltzmann machine (RBM) An example of the binary RBM [1] learning the MNIST data. The RBM is implemented as a custom operator, and a gluon block is also provided. `binary_rbm.py` contains the implementation of the RBM. `binary_rbm_module.py` and `binary_rbm_gluon.py` train the MNIST data using the module interface and the gluon interface respectively. The MNIST data is downloaded automatically. diff --git a/example/rnn/README.md b/example/rnn/README.md index 1d1df6ed7687..cbafc19e2ef0 100644 --- a/example/rnn/README.md +++ b/example/rnn/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Recurrent Neural Network Examples =========== diff --git a/example/rnn/bucketing/README.md b/example/rnn/bucketing/README.md index 7b7883d79ad1..707370af5a96 100644 --- a/example/rnn/bucketing/README.md +++ b/example/rnn/bucketing/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + RNN Example =========== This folder contains RNN examples using high level mxnet.rnn interface. diff --git a/example/rnn/old/README.md b/example/rnn/old/README.md index 5d73523dd964..b199c7770114 100644 --- a/example/rnn/old/README.md +++ b/example/rnn/old/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + RNN Example =========== This folder contains RNN examples using low level symbol interface. diff --git a/example/rnn/word_lm/README.md b/example/rnn/word_lm/README.md index ab0a8d704b9c..9f9e1b75b51e 100644 --- a/example/rnn/word_lm/README.md +++ b/example/rnn/word_lm/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Word Level Language Modeling =========== This example trains a multi-layer LSTM on Sherlock Holmes language modeling benchmark. diff --git a/example/sparse/README.md b/example/sparse/README.md new file mode 100644 index 000000000000..8f1302950d22 --- /dev/null +++ b/example/sparse/README.md @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + +# Examples using Sparse Symbol API +This folder contains examples that demonstrate the usage of [Sparse Symbol API](https://mxnet.incubator.apache.org/api/python/symbol/sparse.html) +- [Factorization Machine](https://github.com/apache/incubator-mxnet/tree/master/example/sparse/factorization_machine) uses sparse weights +- [Linear Classification Using Sparse Matrix Multiplication](https://github.com/apache/incubator-mxnet/tree/master/example/sparse/linear_classification) shows how to use a sparse data loader, sparse dot operator and sparse gradient updaters +- [Matrix Factorization w/ Sparse Embedding](https://github.com/apache/incubator-mxnet/tree/master/example/sparse/matrix_factorization) uses sparse weights +- [Wide and Deep Learning](https://github.com/apache/incubator-mxnet/tree/master/example/sparse/wide_deep) shows how to run sparse wide and deep classification + diff --git a/example/sparse/factorization_machine/README.md b/example/sparse/factorization_machine/README.md index 32b956ed0201..0023b33a1413 100644 --- a/example/sparse/factorization_machine/README.md +++ b/example/sparse/factorization_machine/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Factorization Machine =========== This example trains a factorization machine model using the criteo dataset. diff --git a/example/sparse/linear_classification/README.md b/example/sparse/linear_classification/README.md index 926d9234269d..05674cd0c277 100644 --- a/example/sparse/linear_classification/README.md +++ b/example/sparse/linear_classification/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Linear Classification Using Sparse Matrix Multiplication =========== This examples trains a linear model using the sparse feature in MXNet. This is for demonstration purpose only. diff --git a/example/sparse/matrix_factorization/README.md b/example/sparse/matrix_factorization/README.md index ddbf662c858f..88ae7b8e7678 100644 --- a/example/sparse/matrix_factorization/README.md +++ b/example/sparse/matrix_factorization/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Matrix Factorization w/ Sparse Embedding =========== The example demonstrates the basic usage of the sparse.Embedding operator in MXNet, adapted based on @leopd's recommender examples. diff --git a/example/sparse/wide_deep/README.md b/example/sparse/wide_deep/README.md index 3df5e420ee36..d0ae8ad65825 100644 --- a/example/sparse/wide_deep/README.md +++ b/example/sparse/wide_deep/README.md @@ -1,7 +1,27 @@ + + + + + + + + + + + + + + + + + ## Wide and Deep Learning The example demonstrates how to train [wide and deep model](https://arxiv.org/abs/1606.07792). The [Census Income Data Set](https://archive.ics.uci.edu/ml/datasets/Census+Income) that this example uses for training is hosted by the [UC Irvine Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/). Tricks of feature engineering are adapted from tensorflow's [wide and deep tutorial](https://github.com/tensorflow/models/tree/master/official/wide_deep). The final accuracy should be around 85%. - +For training: - `python train.py` + +For inference: +- `python inference.py` diff --git a/example/sparse/wide_deep/config.py b/example/sparse/wide_deep/config.py new file mode 100644 index 000000000000..c0d20c49f981 --- /dev/null +++ b/example/sparse/wide_deep/config.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Related to feature engineering, please see preprocess in data.py +ADULT = { + 'train': 'adult.data', + 'test': 'adult.test', + 'url': 'https://archive.ics.uci.edu/ml/machine-learning-databases/adult/', + 'num_linear_features': 3000, + 'num_embed_features': 2, + 'num_cont_features': 38, + 'embed_input_dims': [1000, 1000], + 'hidden_units': [8, 50, 100], +} diff --git a/example/sparse/wide_deep/inference.py b/example/sparse/wide_deep/inference.py new file mode 100644 index 000000000000..e14396e50c15 --- /dev/null +++ b/example/sparse/wide_deep/inference.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +from mxnet.test_utils import * +from config import * +from data import get_uci_adult +from model import wide_deep_model +import argparse +import os +import time + +parser = argparse.ArgumentParser(description="Run sparse wide and deep inference", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-infer-batch', type=int, default=100, + help='number of batches to inference') +parser.add_argument('--load-epoch', type=int, default=0, + help='loading the params of the corresponding training epoch.') +parser.add_argument('--batch-size', type=int, default=100, + help='number of examples per batch') +parser.add_argument('--benchmark', action='store_true', default=False, + help='run the script for benchmark mode, not set for accuracy test.') +parser.add_argument('--verbose', action='store_true', default=False, + help='accurcy for each batch will be logged if set') +parser.add_argument('--gpu', action='store_true', default=False, + help='Inference on GPU with CUDA') +parser.add_argument('--model-prefix', type=str, default='checkpoint', + help='the model prefix') + +if __name__ == '__main__': + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + # arg parser + args = parser.parse_args() + logging.info(args) + num_iters = args.num_infer_batch + batch_size = args.batch_size + benchmark = args.benchmark + verbose = args.verbose + model_prefix = args.model_prefix + load_epoch = args.load_epoch + ctx = mx.gpu(0) if args.gpu else mx.cpu() + # dataset + data_dir = os.path.join(os.getcwd(), 'data') + val_data = os.path.join(data_dir, ADULT['test']) + val_csr, val_dns, val_label = get_uci_adult(data_dir, ADULT['test'], ADULT['url']) + # load parameters and symbol + sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, load_epoch) + # data iterator + eval_data = mx.io.NDArrayIter({'csr_data': val_csr, 'dns_data': val_dns}, + {'softmax_label': val_label}, batch_size, + shuffle=True, last_batch_handle='discard') + # module + mod = mx.mod.Module(symbol=sym, context=ctx, data_names=['csr_data', 'dns_data'], + label_names=['softmax_label']) + mod.bind(data_shapes=eval_data.provide_data, label_shapes=eval_data.provide_label) + # get the sparse weight parameter + mod.set_params(arg_params=arg_params, aux_params=aux_params) + + data_iter = iter(eval_data) + nbatch = 0 + if benchmark: + logging.info('Inference benchmark started ...') + tic = time.time() + for i in range(num_iters): + try: + batch = data_iter.next() + except StopIteration: + data_iter.reset() + else: + mod.forward(batch, is_train=False) + for output in mod.get_outputs(): + output.wait_to_read() + nbatch += 1 + score = (nbatch*batch_size)/(time.time() - tic) + logging.info('batch size %d, process %s samples/s' % (batch_size, score)) + else: + logging.info('Inference started ...') + # use accuracy as the metric + metric = mx.metric.create(['acc']) + accuracy_avg = 0.0 + for batch in data_iter: + nbatch += 1 + metric.reset() + mod.forward(batch, is_train=False) + mod.update_metric(metric, batch.label) + accuracy_avg += metric.get()[1][0] + if args.verbose: + logging.info('batch %d, accuracy = %s' % (nbatch, metric.get())) + logging.info('averged accuracy on eval set is %.5f' % (accuracy_avg/nbatch)) diff --git a/example/sparse/wide_deep/train.py b/example/sparse/wide_deep/train.py index 6fd81b7fa480..eea70301660d 100644 --- a/example/sparse/wide_deep/train.py +++ b/example/sparse/wide_deep/train.py @@ -17,6 +17,7 @@ import mxnet as mx from mxnet.test_utils import * +from config import * from data import get_uci_adult from model import wide_deep_model import argparse @@ -31,7 +32,7 @@ help='number of examples per batch') parser.add_argument('--lr', type=float, default=0.001, help='learning rate') -parser.add_argument('--cuda', action='store_true', default=False, +parser.add_argument('--gpu', action='store_true', default=False, help='Train on GPU with CUDA') parser.add_argument('--optimizer', type=str, default='adam', help='what optimizer to use', @@ -40,19 +41,6 @@ help='number of batches to wait before logging training status') -# Related to feature engineering, please see preprocess in data.py -ADULT = { - 'train': 'adult.data', - 'test': 'adult.test', - 'url': 'https://archive.ics.uci.edu/ml/machine-learning-databases/adult/', - 'num_linear_features': 3000, - 'num_embed_features': 2, - 'num_cont_features': 38, - 'embed_input_dims': [1000, 1000], - 'hidden_units': [8, 50, 100], -} - - if __name__ == '__main__': import logging head = '%(asctime)-15s %(message)s' @@ -66,7 +54,7 @@ optimizer = args.optimizer log_interval = args.log_interval lr = args.lr - ctx = mx.gpu(0) if args.cuda else mx.cpu() + ctx = mx.gpu(0) if args.gpu else mx.cpu() # dataset data_dir = os.path.join(os.getcwd(), 'data') @@ -88,7 +76,7 @@ shuffle=True, last_batch_handle='discard') # module - mod = mx.mod.Module(symbol=model, context=ctx ,data_names=['csr_data', 'dns_data'], + mod = mx.mod.Module(symbol=model, context=ctx, data_names=['csr_data', 'dns_data'], label_names=['softmax_label']) mod.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label) mod.init_params() diff --git a/example/speech_recognition/README.md b/example/speech_recognition/README.md index 6f01911e1300..d2125f6578c1 100644 --- a/example/speech_recognition/README.md +++ b/example/speech_recognition/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + **deepSpeech.mxnet: Rich Speech Example** ========================================= diff --git a/example/ssd/README.md b/example/ssd/README.md index 713a9ea33c1b..6d4caa481bd7 100644 --- a/example/ssd/README.md +++ b/example/ssd/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # SSD: Single Shot MultiBox Object Detector SSD is an unified framework for object detection with a single network. diff --git a/example/ssd/dataset/pycocotools/README.md b/example/ssd/dataset/pycocotools/README.md old mode 100755 new mode 100644 index d358f53105da..ed4411425f8c --- a/example/ssd/dataset/pycocotools/README.md +++ b/example/ssd/dataset/pycocotools/README.md @@ -1,2 +1,19 @@ + + + + + + + + + + + + + + + + + This is a modified version of https://github.com/pdollar/coco python API. No `make` is required, but this will not support mask functions. diff --git a/example/ssd/model/README.md b/example/ssd/model/README.md index e5bac52f5a83..7c77ee2475e8 100644 --- a/example/ssd/model/README.md +++ b/example/ssd/model/README.md @@ -1 +1,18 @@ + + + + + + + + + + + + + + + + + #### This is the default directory to store all the models, including `*.params` and `*.json` diff --git a/example/ssd/symbol/README.md b/example/ssd/symbol/README.md index 8fee31985a0d..d577b7067c92 100644 --- a/example/ssd/symbol/README.md +++ b/example/ssd/symbol/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## How to compose SSD network on top of mainstream classification networks 1. Have the base network ready in this directory as `name.py`, such as `inceptionv3.py`. diff --git a/example/ssd/tools/caffe_converter/README.md b/example/ssd/tools/caffe_converter/README.md index 2e74fc56e022..3d4ab13f4694 100644 --- a/example/ssd/tools/caffe_converter/README.md +++ b/example/ssd/tools/caffe_converter/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Convert Caffe Model to Mxnet Format This folder contains the source codes for this tool. diff --git a/example/stochastic-depth/README.md b/example/stochastic-depth/README.md index 08c466eb8b0e..4a41199893ac 100644 --- a/example/stochastic-depth/README.md +++ b/example/stochastic-depth/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Stochastic Depth ================ diff --git a/example/svm_mnist/README.md b/example/svm_mnist/README.md index 408f5108b44a..8d878fb76859 100644 --- a/example/svm_mnist/README.md +++ b/example/svm_mnist/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Use case with Support Vector Machine To ensure that not only the implementation is learning, but is able to outsmart the softmax, as [this article](https://arxiv.org/pdf/1306.0239.pdf) suggests, I ran svm_mnist.py script. It was based on the MNIST experiment description on the article and [this tutorial](https://github.com/dmlc/mxnet-gtc-tutorial/blob/master/tutorial.ipynb). diff --git a/example/svrg_module/README.md b/example/svrg_module/README.md index 250995a57152..f77ecf8fd39d 100644 --- a/example/svrg_module/README.md +++ b/example/svrg_module/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## SVRGModule Example SVRGModule is an extension to the Module API that implements SVRG optimization, which stands for Stochastic diff --git a/example/vae-gan/README.md b/example/vae-gan/README.md index 469668b9b374..17e5b0e68e76 100644 --- a/example/vae-gan/README.md +++ b/example/vae-gan/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # VAE-GAN in MXNet * Implementation of [Autoencoding beyond pixels using a learned similarity metric](https://arxiv.org/abs/1512.09300), diff --git a/include/mkldnn b/include/mkldnn new file mode 120000 index 000000000000..ed48bf601bd6 --- /dev/null +++ b/include/mkldnn @@ -0,0 +1 @@ +../3rdparty/mkldnn/include \ No newline at end of file diff --git a/include/mxnet/base.h b/include/mxnet/base.h index 7f12643119f7..b239cb1f7302 100644 --- a/include/mxnet/base.h +++ b/include/mxnet/base.h @@ -33,9 +33,9 @@ #include "mshadow/tensor.h" // nnvm headers for symbolic construction. #include "nnvm/op.h" -#include "nnvm/tuple.h" #include "nnvm/symbolic.h" -#include "mxfeatures.h" +#include "libinfo.h" +#include "tuple.h" /*! @@ -95,8 +95,6 @@ typedef mshadow::gpu gpu; typedef mshadow::index_t index_t; /*! \brief data type that will be used to store ndarray */ typedef mshadow::default_real_t real_t; -/*! \brief Shape data structure used to record shape information */ -using TShape = nnvm::TShape; /*! \brief operator structure from NNVM */ using Op = nnvm::Op; @@ -193,6 +191,11 @@ struct Context { * \return The number of GPUs that are available. */ inline static int32_t GetGPUCount(); + /*! + * Get the number of streams that a GPU Worker has available to operations. + * \return The number of streams that are available. + */ + inline static int32_t GetGPUStreamsPerWorker(); /*! * \brief get the free and total available memory on a GPU * \param dev the GPU number to query @@ -221,6 +224,112 @@ struct Context { inline static Context FromString(const std::string& str); }; +#if MXNET_USE_CUDA +/*! \brief Holds an auxiliary mshadow gpu stream that can be synced with a primary stream. */ +class GPUAuxStream { + public: + /*! + * \brief constructor. + * \param primary_stream gpu stream that is synced with the created auxiliary stream. + */ + explicit GPUAuxStream(mshadow::Stream *primary_stream) : + primary_stream_(primary_stream), + aux_stream_(primary_stream), + gpu_stream_sync_event_(nullptr) { + if (Context::GetGPUStreamsPerWorker() >= 2) { + // Create auxiliary stream on the same device with the same properties as the primary stream + bool primary_has_blas_handle = + primary_stream->blas_handle_ownership_ == mshadow::Stream::OwnHandle; + bool primary_has_dnn_handle = + primary_stream->dnn_handle_ownership_ == mshadow::Stream::OwnHandle; + aux_stream_ = mshadow::NewStream(primary_has_blas_handle, + primary_has_dnn_handle, + primary_stream->dev_id); + MSHADOW_CUDA_CALL(cudaEventCreateWithFlags(&gpu_stream_sync_event_, cudaEventDisableTiming)); + } + } + /*! \brief destructor */ + ~GPUAuxStream() { + // If the aux_stream_ == primary_stream_, then we created no new streams to destroy. + if (aux_stream_ != primary_stream_) { + MSHADOW_CATCH_ERROR(mshadow::DeleteStream(aux_stream_)); + MSHADOW_CATCH_ERROR(cudaEventDestroy(gpu_stream_sync_event_)); + } + } + /*! + * \brief Makes future aux stream work wait on the completion of existing primary stream work. + */ + void PreAuxStreamUseSync() { + // If the aux_stream_ == primary_stream_, then no synchronization is necessary. + if (aux_stream_ != primary_stream_) + StreamSync(primary_stream_, aux_stream_, gpu_stream_sync_event_); + } + /*! + * \brief Makes future primary stream work wait on the completion of existing aux stream work. + */ + void PostAuxStreamUseSync() { + // If the aux_stream_ == primary_stream_, then no synchronization is necessary. + if (aux_stream_ != primary_stream_) + StreamSync(aux_stream_, primary_stream_, gpu_stream_sync_event_); + } + /*! \brief Getter for created auxiliary stream. */ + mshadow::Stream *GetStream() { return aux_stream_; } + /*! + * \brief Make future work enqueued to `s2` wait on completion of current work enqueued to `s1`. + * \param s1 stream with work that must be completed before future s2 work can begin. + * \param s2 stream whose future work is made to wait on the completion of existing s1 work. + * \param event used to pass s1 state to s2. + */ + static void StreamSync(mshadow::Stream *s1, mshadow::Stream *s2, cudaEvent_t event) { + MSHADOW_CUDA_CALL(cudaEventRecord(event, s1->stream_)); + MSHADOW_CUDA_CALL(cudaStreamWaitEvent(s2->stream_, event, 0)); + } + + private: + mshadow::Stream *primary_stream_; + mshadow::Stream *aux_stream_; + cudaEvent_t gpu_stream_sync_event_; +}; + +/*! + * \brief Provides automatic coordination of an auxilary stream with a primary one. + * This object, upon construction, prepares an aux stream for use by syncing it with enqueued + * primary-stream work. Object destruction will sync again so future primary-stream work + * will wait on enqueued aux-stream work. If MXNET_GPU_WORKER_NSTREAMS == 1, then this defaults + * simply: the primary stream will equal the aux stream and the syncs will be executed as nops. + * See ./src/operator/cudnn/cudnn_convolution-inl.h for a usage example. + */ +class SyncedGPUAuxStream { + public: + /*! + * \brief constructor. + * \param gpu_aux_stream auxilary gpu stream that is managed by this RAII object. + */ + explicit SyncedGPUAuxStream(GPUAuxStream *gpu_aux_stream) : gpu_aux_stream_(gpu_aux_stream) { + gpu_aux_stream_->PreAuxStreamUseSync(); + } + /*! \brief destructor */ + ~SyncedGPUAuxStream() { + gpu_aux_stream_->PostAuxStreamUseSync(); + } + /*! \brief copy constructor deleted to prevent unexpected synchronizations. */ + SyncedGPUAuxStream(const SyncedGPUAuxStream&) = delete; + /*! \brief copy assignment operator deleted to prevent unexpected synchronizations. */ + void operator=(const SyncedGPUAuxStream&) = delete; + /*! \brief move constructor permitted as alternative to copying. */ + SyncedGPUAuxStream(SyncedGPUAuxStream&&) = default; + /*! \brief move assignment operator permitted as alternative to copy assignment. */ + SyncedGPUAuxStream& operator=(SyncedGPUAuxStream&&) = default; + /*! \brief Getter for underlying mshadow::Stream. */ + inline mshadow::Stream* GetStream() const { + return gpu_aux_stream_->GetStream(); + } + + private: + GPUAuxStream *gpu_aux_stream_; +}; +#endif // MXNET_USE_CUDA + /*! * \brief execution time context. * The information needed in runtime for actual execution. @@ -232,6 +341,10 @@ struct RunContext { * \brief the stream of the device, can be NULL or Stream* in GPU mode */ void *stream; + /*! + * \brief the auxiliary stream of the device, can be NULL or Stream* in GPU mode + */ + void *aux_stream; /*! * \brief indicator of whether this execution is run in bulk mode */ @@ -245,6 +358,15 @@ struct RunContext { inline mshadow::Stream* get_stream() const { return static_cast*>(stream); } +#if MXNET_USE_CUDA + /*! + * \brief get an RAII object that transparently handles the syncing of the auxiliary stream. + * \return the aux stream auto-syncing object + */ + inline SyncedGPUAuxStream get_gpu_aux_stream() const { + return SyncedGPUAuxStream(static_cast(aux_stream)); + } +#endif /*! \brief get the base Context from RunContext */ inline const Context& get_ctx() const { return ctx; @@ -309,6 +431,15 @@ inline int32_t Context::GetGPUCount() { #endif } +inline int32_t Context::GetGPUStreamsPerWorker() { + // The default number of streams available if the user has not set MXNET_GPU_WORKER_NSTREAMS. + const int32_t default_num_streams = 1; + // The get_aux_stream() interface can supply one additional stream beyond the standard one. + static int32_t num_streams = + dmlc::GetEnv("MXNET_GPU_WORKER_NSTREAMS", default_num_streams) >= 2 ? 2 : 1; + return num_streams; +} + inline void Context::GetGPUMemoryInformation(int dev, uint64_t *free_mem, uint64_t *total_mem) { #if MXNET_USE_CUDA @@ -403,7 +534,14 @@ template<> struct hash { return res; } }; + +#if __cplusplus < 201402L && !defined(_MSC_VER) +template +inline std::unique_ptr make_unique(Args&&... args) { + return std::unique_ptr(new T(std::forward(args)...)); } +#endif +} // namespace std #include "./tensor_blob.h" //! \endcond diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index d6e13ebcf051..9a24b7516128 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -139,6 +139,11 @@ struct MXCallbackList { void **contexts; }; +struct LibFeature { + const char* name; + bool enabled; +}; + enum CustomOpCallbacks { kCustomOpDelete, kCustomOpForward, @@ -210,12 +215,12 @@ MXNET_DLL const char *MXGetLastError(); //------------------------------------- /*! - * \brief - * \param feature to check mxfeatures.h - * \param out set to true if the feature is enabled, false otherwise + * \brief Get list of features supported on the runtime + * \param libFeature pointer to array of LibFeature + * \param size of the array * \return 0 when success, -1 when failure happens. */ -MXNET_DLL int MXHasFeature(const mx_uint feature, bool* out); +MXNET_DLL int MXLibInfoFeatures(const struct LibFeature **libFeature, size_t *size); /*! * \brief Seed all global random number generators in mxnet. @@ -1557,6 +1562,38 @@ MXNET_DLL int MXSymbolInferType(SymbolHandle sym, const int **aux_type_data, int *complete); +/*! + * \brief partially infer type of unknown input types given the known one. + * + * Return partially inferred results if not all types could be inferred. + * The types are packed into a CSR matrix represented by arg_ind_ptr and arg_type_data + * The call will be treated as a kwargs call if key != nullptr or num_args==0, otherwise it is positional. + * + * \param sym symbol handle + * \param num_args numbe of input arguments. + * \param keys the key of keyword args (optional) + * \param arg_type_data the content of the CSR + * \param in_type_size sizeof the returning array of in_types + * \param in_type_data returning array of pointers to head of the input type. + * \param out_type_size sizeof the returning array of out_types + * \param out_type_data returning array of pointers to head of the input type. + * \param aux_type_size sizeof the returning array of aux_types + * \param aux_type_data returning array of pointers to head of the auxiliary type. + * \param complete whether infer type completes or more information is needed. + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXSymbolInferTypePartial(SymbolHandle sym, + mx_uint num_args, + const char** keys, + const int *arg_type_data, + mx_uint *in_type_size, + const int **in_type_data, + mx_uint *out_type_size, + const int **out_type_data, + mx_uint *aux_type_size, + const int **aux_type_data, + int *complete); + /*! * \brief Convert a symbol into a quantized symbol where FP32 operators are replaced with INT8 * \param sym_handle symbol to be converted @@ -1565,8 +1602,8 @@ MXNET_DLL int MXSymbolInferType(SymbolHandle sym, * \param excluded_symbols op names to be excluded from being quantized * \param num_offline number of parameters that are quantized offline * \param offline_params array of c strings representing the names of params quantized offline - * \param quantized_dtype the quantized destination type for input data. - * \param calib_quantize **Deprecated**. quantize op will always be calibrated if could. + * \param quantized_dtype the quantized destination type for input data + * \param calib_quantize **Deprecated**. quantize op will always be calibrated if could */ MXNET_DLL int MXQuantizeSymbol(SymbolHandle sym_handle, SymbolHandle *ret_sym_handle, const mx_uint num_excluded_symbols, diff --git a/include/mxnet/executor.h b/include/mxnet/executor.h index aec10091a540..24b23ed783f3 100644 --- a/include/mxnet/executor.h +++ b/include/mxnet/executor.h @@ -121,7 +121,7 @@ class Executor { const bool allow_up_sizing, const Context& default_ctx, const std::map& ctx_map, - const std::unordered_map& + const std::unordered_map& provided_arg_shapes, std::vector* in_args, std::vector* arg_grads, @@ -155,7 +155,7 @@ class Executor { const std::vector& in_arg_ctxes, const std::vector& arg_grad_ctxes, const std::vector& aux_state_ctxes, - const std::unordered_map& arg_shape_map, + const std::unordered_map& arg_shape_map, const std::unordered_map& arg_dtype_map, const std::unordered_map& arg_stype_map, const std::vector& grad_req_types, diff --git a/include/mxnet/mxfeatures.h b/include/mxnet/libinfo.h similarity index 82% rename from include/mxnet/mxfeatures.h rename to include/mxnet/libinfo.h index 10f9b3656692..f35d41a9aa8a 100644 --- a/include/mxnet/mxfeatures.h +++ b/include/mxnet/libinfo.h @@ -18,21 +18,27 @@ */ /*! - * Copyright (c) 2018 by Contributors - * \file mxfeatures.h - * \brief check MXNet features including compile time support + * Copyright (c) 2018 by Contributors + * \file libinfo.h + * \author larroy + * \brief get features of the MXNet library at runtime */ #pragma once +#include +#include +#include +#include #include "dmlc/base.h" #include "mshadow/base.h" +#include "c_api.h" /*! *\brief whether to use opencv support */ #ifndef MXNET_USE_OPENCV -#define MXNET_USE_OPENCV 1 +#define MXNET_USE_OPENCV 0 #endif /*! @@ -124,7 +130,8 @@ namespace features { // Check compile flags such as CMakeLists.txt /// Compile time features -enum : uint32_t { +// ATTENTION: When changing this enum, match the strings in the implementation file! +enum : unsigned { // NVIDIA, CUDA CUDA = 0, CUDNN, @@ -179,10 +186,25 @@ enum : uint32_t { }; +struct EnumNames { + static const std::vector names; +}; + +struct LibInfo { + LibInfo(); + static LibInfo* getInstance(); + const std::array& getFeatures() { + return m_lib_features; + } + private: + std::array m_lib_features; + static std::unique_ptr m_inst; +}; + /*! * \return true if the given feature is supported */ -bool is_enabled(uint32_t feat); +bool is_enabled(unsigned feat); } // namespace features } // namespace mxnet diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index 5de42e19a657..c55cb01b4688 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -91,7 +91,7 @@ class NDArray { * \param delay_alloc whether delay the allocation * \param dtype data type of this ndarray */ - NDArray(const TShape &shape, Context ctx, + NDArray(const mxnet::TShape &shape, Context ctx, bool delay_alloc = false, int dtype = mshadow::default_type_flag) : ptr_(std::make_shared(shape, ctx, delay_alloc, dtype)), shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), @@ -99,10 +99,10 @@ class NDArray { } /*! \brief constructor for NDArray with storage type */ - NDArray(const NDArrayStorageType stype, const TShape &shape, Context ctx, + NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, Context ctx, bool delay_alloc = true, int dtype = mshadow::default_type_flag, - std::vector aux_types = {}, std::vector aux_shapes = {}, - TShape storage_shape = TShape(mshadow::Shape1(0))); + std::vector aux_types = {}, mxnet::ShapeVector aux_shapes = {}, + mxnet::TShape storage_shape = mxnet::TShape(mshadow::Shape1(0))); /*! * \brief constructs a new dynamic NDArray whose shape is unknown, * hence the NDArray is inherently lazily created @@ -110,7 +110,7 @@ class NDArray { * \param dtype data type of this ndarray */ explicit NDArray(Context ctx, int dtype = mshadow::default_type_flag) { - ptr_ = std::make_shared(TShape(mshadow::Shape1(0)), ctx, true, dtype); + ptr_ = std::make_shared(mxnet::TShape(mshadow::Shape1(0)), ctx, true, dtype); dtype_ = dtype; storage_type_ = kDefaultStorage; entry_ = {nullptr, 0, 0}; @@ -148,7 +148,7 @@ class NDArray { } /*! \brief create ndarray from shared memory */ - NDArray(int shared_pid, int shared_id, const TShape& shape, int dtype) + NDArray(int shared_pid, int shared_id, const mxnet::TShape& shape, int dtype) : ptr_(std::make_shared(shared_pid, shared_id, shape, dtype)), shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), entry_({nullptr, 0, 0}) { } @@ -163,7 +163,7 @@ class NDArray { * \param aux_data the memory content of static aux data * \param dev_id the device id this tensor sits at */ - NDArray(const NDArrayStorageType stype, const TShape &shape, + NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, const TBlob &data, const std::vector &aux_data, int dev_id) : ptr_(std::make_shared(stype, data, aux_data, dev_id)), shape_(shape), dtype_(data.type_flag_), storage_type_(stype), entry_({nullptr, 0, 0}) { @@ -172,7 +172,7 @@ class NDArray { * \brief initialize the NDArray, assuming it is not assigned a meaningful shape before * \param shape the shape of the NDArray */ - void Init(const TShape &shape) { + void Init(const mxnet::TShape &shape) { ptr_->Init(shape, this->dtype_); this->shape_ = shape; } @@ -210,7 +210,7 @@ class NDArray { /*! * \return the shape of current NDArray. */ - inline const TShape& shape() const { + inline const mxnet::TShape& shape() const { return shape_; } /*! @@ -218,7 +218,7 @@ class NDArray { * It is only intended for non-default storage. For row-sparse storage, it is the shape of * the tensor which stores the non-zero values. */ - inline const TShape &storage_shape() const { + inline const mxnet::TShape &storage_shape() const { CHECK(ptr_ != nullptr); CHECK_NE(storage_type(), kDefaultStorage) << "storage_shape() is not intended for kDefaultStorage."; @@ -230,14 +230,14 @@ class NDArray { * \param index the index of the aux data * \return the shape of aux data at given index */ - inline const TShape& aux_shape(size_t index) const { + inline const mxnet::TShape& aux_shape(size_t index) const { CHECK_NE(storage_type(), kDefaultStorage) << "aux_shape() is not intended for kDefaultStorage."; return ptr_->aux_shapes[index]; } /* \return the shapes of all aux data */ - const std::vector& aux_shapes() const { + const mxnet::ShapeVector& aux_shapes() const { CHECK_NE(storage_type(), kDefaultStorage) << "aux_shapes() is not intended for kDefaultStorage."; return ptr_->aux_shapes; @@ -257,7 +257,7 @@ class NDArray { * for the final result. After the operation is done, the exact size of * the shape is known and need to be reset using this function. */ - inline void set_aux_shape(size_t index, const TShape& shape) const { + inline void set_aux_shape(size_t index, const mxnet::TShape& shape) const { CHECK_NE(storage_type(), kDefaultStorage) << "set_aux_shape() is not intended for kDefaultStorage."; ptr_->set_aux_shape(index, shape); @@ -552,7 +552,7 @@ class NDArray { * \param dtype The data type. * \return NDArray in new shape and type. */ - inline NDArray AsArray(const TShape &shape, int dtype) const { + inline NDArray AsArray(const mxnet::TShape &shape, int dtype) const { CHECK_EQ(storage_type(), kDefaultStorage) << "AsArray is intended only for kDefaultStorage."; CHECK_GE(ptr_->shandle.size, @@ -628,13 +628,13 @@ class NDArray { * \param shape new shape * \return NDArray in new shape */ - NDArray Reshape(const TShape &shape) const; + NDArray Reshape(const mxnet::TShape &shape) const; /*! * \brief Get an reshaped NDArray. Supports autograd recording * \param shape new shape * \return NDArray in new shape */ - NDArray ReshapeWithRecord(const TShape &shape); + NDArray ReshapeWithRecord(const mxnet::TShape &shape); /*! * \brief Return a copy of this NDArray without autograd history */ @@ -660,10 +660,10 @@ class NDArray { * This function can only be called by ndarray of default * storage type and effectively changes the ndarray's shape_. * Note: This function is named as this to avoid overload conflict - * with CheckAndAlloc(const std::vector &aux_shapes), since - * TShape tmp = some_shape is equivalent to TShape tmp = {some_shape}. + * with CheckAndAlloc(const mxnet::ShapeVector &aux_shapes), since + * mxnet::TShape tmp = some_shape is equivalent to mxnet::TShape tmp = {some_shape}. */ - void ReshapeAndAlloc(const TShape& shape) { + void ReshapeAndAlloc(const mxnet::TShape& shape) { CHECK_EQ(storage_type(), kDefaultStorage); CHECK(!is_none()); shape_ = shape; @@ -674,17 +674,17 @@ class NDArray { * \brief Alloc memory for non-default storage * aux_shape is only known at run time */ - inline void CheckAndAlloc(const std::vector &aux_shapes) const { + inline void CheckAndAlloc(const mxnet::ShapeVector &aux_shapes) const { CHECK_NE(storage_type(), kDefaultStorage) << "CheckAndAlloc(aux_shapes) is not intended for kDefaultStorage"; ptr_->CheckAndAlloc(shape_, aux_shapes, dtype_); } - inline void CheckAndAllocData(const TShape &storage_shape) const { + inline void CheckAndAllocData(const mxnet::TShape &storage_shape) const { CHECK_NE(storage_type(), kDefaultStorage) << "CheckAndAllocData is not intended for kDefaultStorage"; ptr_->CheckAndAllocData(storage_shape, dtype_); } - inline void CheckAndAllocAuxData(size_t i, const TShape &aux_shape) const { + inline void CheckAndAllocAuxData(size_t i, const mxnet::TShape &aux_shape) const { CHECK_NE(storage_type(), kDefaultStorage) << "CheckAndAllocAuxData is not intended for kDefaultStorage"; ptr_->CheckAndAllocAuxData(i, aux_shape); @@ -775,7 +775,7 @@ class NDArray { * which can be expensive. * It's used by FullyConnected right now. */ - NDArray MKLDNNDataReshape(const TShape &shape) const; + NDArray MKLDNNDataReshape(const mxnet::TShape &shape) const; /*! * \ Fix mkldnn memory descriptor mismatch from NDArray. @@ -844,17 +844,21 @@ class NDArray { // The shape of the chunk data. // This might not be the same shape as the NDArray, since the storage may be sparse. // The default value for storage_shape is {0} when an empty non-default NDArray is created. - TShape storage_shape; + mxnet::TShape storage_shape; // The shape of aux data. The default value for the shape depends on the type of storage. // If aux_shapes[i].Size() is zero, aux data i is empty. - std::vector aux_shapes; + mxnet::ShapeVector aux_shapes; + /*! \brief Reference to the storage to ensure proper destruct order */ + std::shared_ptr storage_ref_; /*! \brief default cosntructor */ - Chunk() : static_data(true), delay_alloc(false) {} + Chunk() : static_data(true), delay_alloc(false), + storage_ref_(Storage::_GetSharedRef()) {} /*! \brief construct a new chunk */ - Chunk(TShape shape, Context ctx_, bool delay_alloc_, int dtype) - : static_data(false), delay_alloc(true), ctx(ctx_) { + Chunk(mxnet::TShape shape, Context ctx_, bool delay_alloc_, int dtype) + : static_data(false), delay_alloc(true), ctx(ctx_), + storage_ref_(Storage::_GetSharedRef()) { auto size = shape.Size(); storage_shape = shape; var = Engine::Get()->NewVariable(); @@ -864,7 +868,8 @@ class NDArray { } Chunk(const TBlob &data, int dev_id) - : static_data(true), delay_alloc(false) { + : static_data(true), delay_alloc(false), + storage_ref_(Storage::_GetSharedRef()) { CHECK(storage_type == kDefaultStorage); var = Engine::Get()->NewVariable(); if (data.dev_mask() == cpu::kDevMask) { @@ -880,8 +885,9 @@ class NDArray { storage_shape = data.shape_; } - Chunk(int shared_pid, int shared_id, const TShape& shape, int dtype) - : static_data(false), delay_alloc(false) { + Chunk(int shared_pid, int shared_id, const mxnet::TShape& shape, int dtype) + : static_data(false), delay_alloc(false), + storage_ref_(Storage::_GetSharedRef()) { var = Engine::Get()->NewVariable(); ctx = Context::CPUShared(0); shandle.size = shape.Size() * mshadow::mshadow_sizeof(dtype); @@ -892,12 +898,12 @@ class NDArray { storage_shape = shape; } // Constructor for a non-default storage chunk - Chunk(NDArrayStorageType storage_type_, const TShape &storage_shape_, Context ctx_, + Chunk(NDArrayStorageType storage_type_, const mxnet::TShape &storage_shape_, Context ctx_, bool delay_alloc_, int dtype, const std::vector &aux_types_, - const std::vector &aux_shapes_) + const mxnet::ShapeVector &aux_shapes_) : static_data(false), delay_alloc(delay_alloc_), storage_type(storage_type_), aux_types(aux_types_), ctx(ctx_), storage_shape(storage_shape_), - aux_shapes(aux_shapes_) { + aux_shapes(aux_shapes_), storage_ref_(Storage::_GetSharedRef()) { shandle.ctx = ctx; var = Engine::Get()->NewVariable(); // aux_handles always reflect the correct number of aux data @@ -914,7 +920,8 @@ class NDArray { Chunk(const NDArrayStorageType storage_type_, const TBlob &data, const std::vector &aux_data, int dev_id) - : static_data(true), delay_alloc(false), storage_type(storage_type_) { + : static_data(true), delay_alloc(false), storage_type(storage_type_), + storage_ref_(Storage::_GetSharedRef()) { using namespace mshadow; CHECK_NE(storage_type, kDefaultStorage); // init var @@ -944,7 +951,7 @@ class NDArray { } /*! \brief set the shape for ith aux data, and update storage shape if necessary */ - inline void set_aux_shape(const size_t i, const TShape& shape) { + inline void set_aux_shape(const size_t i, const mxnet::TShape& shape) { aux_shapes[i] = shape; if (storage_shape.ndim() > 0) { if (storage_type == kRowSparseStorage && i == rowsparse::kIdx) { @@ -989,20 +996,20 @@ class NDArray { } } /*! \brief initialize the shape and dtype, assuming it is not initialized before. */ - void Init(const TShape &shape, int dtype) { + void Init(const mxnet::TShape &shape, int dtype) { auto size = shape.Size(); storage_shape = shape; shandle.size = size * mshadow::mshadow_sizeof(dtype); this->CheckAndAlloc(); } - inline void CheckAndAlloc(const TShape &shape, const std::vector &aux_shapes, + inline void CheckAndAlloc(const mxnet::TShape &shape, const mxnet::ShapeVector &aux_shapes, int dtype) { // calculate size, perform allocation if (kRowSparseStorage == storage_type) { // For row sparse, aux_shape indicates the number of rows to allocate auto aux_shape = aux_shapes[rowsparse::kIdx]; CheckAndAllocAuxData(rowsparse::kIdx, aux_shape); - TShape storage_shape(shape); + mxnet::TShape storage_shape(shape); storage_shape[0] = aux_shape[0]; CheckAndAllocData(storage_shape, dtype); } else if (kCSRStorage == storage_type) { @@ -1017,12 +1024,12 @@ class NDArray { // storage shape is also updated // if data is already allocated, try reuse the storage. Otherwise, free the current one // and allocate new storage - void CheckAndAllocData(const TShape &shape, int dtype); + void CheckAndAllocData(const mxnet::TShape &shape, int dtype); #if MXNET_USE_MKLDNN == 1 // Have MKL memory reference to the data in the default storage // or create memory for MKLDNN. - void SetMKLMem(const TShape &shape, int dtype); + void SetMKLMem(const mxnet::TShape &shape, int dtype); // If the data is stored in MKLDNN layout, we reorder data in mkl_mem_ and // save the result in shandle. void Reorder2Default(); @@ -1037,7 +1044,7 @@ class NDArray { // aux shape is also updated // if aux data is already allocated, try reuse the storage. Otherwise, free the current one // and allocate new storage - inline void CheckAndAllocAuxData(size_t i, const TShape &shape) { + inline void CheckAndAllocAuxData(size_t i, const mxnet::TShape &shape) { CHECK_EQ(shape.ndim(), 1) << "shape must be 1D in CheckAndAllocAuxData"; CHECK_NE(storage_type, kUndefinedStorage) << "storage type cannot be kUndefinedStorage in CheckAndAllocAuxData"; @@ -1065,7 +1072,7 @@ class NDArray { /*! \brief internal data of NDArray */ std::shared_ptr ptr_{nullptr}; /*! \brief shape of current NDArray */ - TShape shape_; + mxnet::TShape shape_; /*! \brief byte offset in chunk */ size_t byte_offset_ = 0; /*! \brief type of data */ diff --git a/include/mxnet/op_attr_types.h b/include/mxnet/op_attr_types.h index aba59ce5f0ea..889b5028a460 100644 --- a/include/mxnet/op_attr_types.h +++ b/include/mxnet/op_attr_types.h @@ -83,6 +83,15 @@ struct OpContext { inline mshadow::Stream* get_stream() const { return run_ctx.get_stream(); } +#if MXNET_USE_CUDA + /*! + * \brief get auxilary gpu stream auto-syncing object from Context + * \return the aux stream auto-syncing object + */ + inline SyncedGPUAuxStream get_gpu_aux_stream() const { + return run_ctx.get_gpu_aux_stream(); + } +#endif }; /*! \brief the execution type of the operator */ @@ -197,7 +206,7 @@ class OpStatePtr { */ using FCreateOpState = std::function& in_shape, + const mxnet::ShapeVector& in_shape, const std::vector& in_type)>; /*! * \brief Execution mode of this operator. diff --git a/include/mxnet/operator.h b/include/mxnet/operator.h index cfa162780495..1dc04244f673 100644 --- a/include/mxnet/operator.h +++ b/include/mxnet/operator.h @@ -50,7 +50,7 @@ namespace mxnet { * To add new operator(aka. layers of neural nets) to mxnet, developer need to create * a new OperatorProperty and its corresponding Operator. * - * \sa TBlob, TShape, OperatorProperty + * \sa TBlob, mxnet::TShape, OperatorProperty */ class Operator { public: @@ -192,15 +192,15 @@ class OperatorProperty { * common practice: set the shape of data input, and usually weight's shape can be inferred * * \param out_shape the shape of outputs of the operator - * InferShape will modify the vector to fill output TShape + * InferShape will modify the vector to fill output mxnet::TShape * \param aux_shape the shape of auxiliary states of the operator - * InferShape will modify the vector to fill output TShape + * InferShape will modify the vector to fill output mxnet::TShape * \return true if the shape inference is successful, false if there is not enough information. * \throws dmlc::Error if the known arg_shapes are inconsistent. */ - virtual bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const = 0; + virtual bool InferShape(mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape, + mxnet::ShapeVector *aux_shape) const = 0; /*! * \brief infer the data types of outputs and unknown input arguments * \param in_type the type of input arguments of the operator @@ -255,10 +255,10 @@ class OperatorProperty { * \param in_type dtype of the input ndarrays * \return the created operator */ - virtual Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + virtual Operator* CreateOperatorEx(Context ctx, mxnet::ShapeVector *in_shape, std::vector *in_type) const { std::vector out_type, aux_type; - std::vector out_shape, aux_shape; + mxnet::ShapeVector out_shape, aux_shape; out_type.resize(this->ListOutputs().size()); out_shape.resize(this->ListOutputs().size()); aux_type.resize(this->ListAuxiliaryStates().size()); @@ -284,7 +284,7 @@ class OperatorProperty { * \return Additional resource request */ virtual std::vector ForwardResource( - const std::vector &in_shape) const { + const mxnet::ShapeVector &in_shape) const { return std::vector(); } /*! @@ -295,7 +295,7 @@ class OperatorProperty { * \return Additional resource request */ virtual std::vector BackwardResource( - const std::vector &in_shape) const { + const mxnet::ShapeVector &in_shape) const { return std::vector(); } /*! diff --git a/include/mxnet/operator_util.h b/include/mxnet/operator_util.h index bebe3f13ae45..941e76e6d70b 100644 --- a/include/mxnet/operator_util.h +++ b/include/mxnet/operator_util.h @@ -97,7 +97,7 @@ typedef void (*SourceFunction)(const EnvArguments& env, * \param env The Environment arguments. * \return The inferred result shape. */ -typedef TShape (*SourceShapeFunction)(const EnvArguments& env); +typedef mxnet::TShape (*SourceShapeFunction)(const EnvArguments& env); /*! * \brief Unary function that takes a src and save result to ret. @@ -119,7 +119,7 @@ typedef void (*UnaryFunction)(const TBlob& src, * \param env The Environment arguments. * \return The inferred result shape. */ -typedef TShape (*UnaryShapeFunction)(const TShape& src, +typedef mxnet::TShape (*UnaryShapeFunction)(const mxnet::TShape& src, const EnvArguments& env); /*! @@ -189,8 +189,8 @@ typedef void (*BinaryFunction)(const TBlob& lhs, * \param env The Environment arguments. * \return The inferred result shape. */ -typedef TShape (*BinaryShapeFunction)(const TShape& lhs, - const TShape& rhs, +typedef mxnet::TShape (*BinaryShapeFunction)(const mxnet::TShape& lhs, + const mxnet::TShape& rhs, const EnvArguments& env); /*! * \brief Gradient function that takes only output gradient and computes gradient wrt to input. diff --git a/include/mxnet/tensor_blob.h b/include/mxnet/tensor_blob.h index 412877a58218..7d059025b03e 100755 --- a/include/mxnet/tensor_blob.h +++ b/include/mxnet/tensor_blob.h @@ -69,7 +69,7 @@ class TBlob { /*! \brief pointer to the data */ void *dptr_; /*! \brief shape of the tensor */ - TShape shape_; + mxnet::TShape shape_; /*! \brief type flag of the tensor blob */ int type_flag_; @@ -87,7 +87,7 @@ class TBlob { * \param dev_id the device id */ template - TBlob(DType *dptr, const TShape &shape, int dev_mask, int dev_id = -1) + TBlob(DType *dptr, const mxnet::TShape &shape, int dev_mask, int dev_id = -1) : dptr_(dptr), shape_(shape), type_flag_(mshadow::DataType::kFlag) { SetDLTensor(dev_mask, dev_id); @@ -100,7 +100,7 @@ class TBlob { * \param type_flag the type flag. Can be one of enum mshadow::dtype * \param dev_id the device id */ - TBlob(void *dptr, const TShape &shape, int dev_mask, int type_flag, int dev_id = -1) + TBlob(void *dptr, const mxnet::TShape &shape, int dev_mask, int type_flag, int dev_id = -1) : dptr_(dptr), shape_(shape), type_flag_(type_flag) { SetDLTensor(dev_mask, dev_id); } @@ -110,7 +110,7 @@ class TBlob { */ explicit TBlob(const DLTensor &dltensor) : dptr_(dltensor.data), - shape_(TShape(dltensor.shape, dltensor.shape + dltensor.ndim)), + shape_(mxnet::TShape(dltensor.shape, dltensor.shape + dltensor.ndim)), type_flag_(DLDataTypeTransform(dltensor.dtype)), dltensor_(dltensor) { // compactness check for DLTensor @@ -175,7 +175,7 @@ class TBlob { * \param shape desired shape * \return reshaped blob */ - inline TBlob reshape(const TShape& shape) const { + inline TBlob reshape(const mxnet::TShape& shape) const { CHECK_EQ(this->shape_.Size(), shape.Size()) << "Shape size mismatch " << this->shape_.Size() << " v.s. " << shape.Size(); TBlob ret(this->dptr_, shape, this->dev_mask(), this->type_flag_, this->dev_id()); @@ -417,7 +417,7 @@ class TBlob { } // namespace mxnet namespace dmlc { -// Add a few patches to support TShape in dmlc/parameter. +// Add a few patches to support mxnet::TShape in dmlc/parameter. DMLC_DECLARE_TYPE_NAME(mxnet::TShape, "Shape(tuple)"); DMLC_DECLARE_TYPE_NAME(nnvm::Tuple, "Shape(tuple)"); DMLC_DECLARE_TYPE_NAME(nnvm::Tuple>, "Shape(tuple)"); diff --git a/include/mxnet/tuple.h b/include/mxnet/tuple.h new file mode 100644 index 000000000000..7c1367333630 --- /dev/null +++ b/include/mxnet/tuple.h @@ -0,0 +1,682 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/*! + * Copyright (c) 2016 by Contributors + * \file mxnet/tuple.h + * \brief Data structure Tuple and TShape to store dynamic sized shapes. + */ +#ifndef MXNET_TUPLE_H_ +#define MXNET_TUPLE_H_ + +#include +#include +#include +#include +#include +#include +#include "nnvm/op_attr_types.h" +#include "nnvm/graph_attr_types.h" +#include "nnvm/graph.h" +#include "nnvm/pass.h" + +namespace mxnet { + +/*! + * \brief A dynamic sized array data structure that is optimized for storing + * small number of elements with same type. + * + * Data will be stored in stack when number of elements is small. + * It is suitable to hold shape of Tensor. + * + * \tparam ValueType The type of data stored inside tuple. + * \sa TShape + */ +template +class Tuple { + public: + /*! \brief default constructor */ + Tuple() = default; + /*! \brief destructor */ + inline ~Tuple() { + delete [] data_heap_; + } + /*! + * \brief copy constructor from another tuple + * \param s the source tuple + */ + inline Tuple(const Tuple& s) { + this->assign(s.begin(), s.end()); + } + /*! + * \brief constructor from initializer list + * \param init the initializer_list + */ + inline Tuple(std::initializer_list init) { + this->assign(init.begin(), init.end()); + } + /*! + * \brief constructor from vector + * \param init the vector + */ + inline Tuple(std::vector init) { // NOLINT(runtime/explicit) + this->assign(init.begin(), init.end()); + } + /*! + * \brief move constructor from Tuple + * \param src the source shape + */ + + inline Tuple(Tuple&& src) { // NOLINT(runtime/explicit) + this->swap(src); + } + /*! + * \brief construct the Tuple from content of iterator + * \param begin the beginning of iterator + * \param end end the end of the iterator + * \tparam RandomAccessIterator iterator type + */ + template + inline Tuple(RandomAccessIterator begin, + RandomAccessIterator end) { + this->assign(begin, end); + } + /*! + * \brief Assign content to tuple from iterator. + * \param begin the beginning of iterator + * \param end end the end of the iterator + * \tparam RandomAccessIterator iterator type + */ + template + inline void assign(RandomAccessIterator begin, + RandomAccessIterator end) { + this->SetDim(end - begin); + std::copy(begin, end, this->begin()); + } + /*! + * \brief Swap current object with other + * \param other another object to be swapped. + */ + inline void swap(Tuple& other) { // NOLINT(*) + std::swap(ndim_, other.ndim_); + std::swap(num_heap_allocated_, other.num_heap_allocated_); + std::swap(data_stack_, other.data_stack_); + std::swap(data_heap_, other.data_heap_); + } + /*! + * \brief assignment from another tuple. + * \param src source tuple + * \return reference of self + */ + inline Tuple& operator=(const Tuple& src) { + this->assign(src.begin(), src.end()); + return *this; + } + /*! + * \brief assignment from rvalue of another tuple. + * \param src source tuple + * \return reference of self + */ + inline Tuple& operator=(Tuple&& src) { + Tuple(std::move(src)).swap(*this); + return *this; + } + /*! + * \brief assignment from initializer list + * \param init the source initializer list + * \return reference of self + */ + inline Tuple &operator=(std::initializer_list init) { + this->assign(init.begin(), init.end()); + return *this; + } + /*! + * \return whether two tuple equals + * \param s the tuple to compare against + */ + inline bool operator==(const Tuple &s) const { + if (ndim_ != s.ndim_) return false; + return std::equal(begin(), end(), s.begin()); + } + /*! + * \return whether two tuple not equal + * \param s the tuple to compare against + */ + inline bool operator!=(const Tuple &s) const { + return !(*this == s); + } + /*! \return the begin data pointer to content of the tuple */ + inline const ValueType *begin() const { + return ndim_ <= kStackCache ? data_stack_ : data_heap_; + } + /*! \return the begin data pointer to content of the tuple */ + inline ValueType *begin() { + return ndim_ <= kStackCache ? data_stack_ : data_heap_; + } + /*! \return the data pointer to end of the tuple */ + inline const ValueType* end() const { + return ndim_ <= kStackCache ? (data_stack_ + ndim_): (data_heap_ + ndim_); + } + /*! \return the data pointer to end the tuple */ + inline ValueType* end() { + return ndim_ <= kStackCache ? (data_stack_ + ndim_): (data_heap_ + ndim_); + } + /*! \return number of dimension of the tuple */ + inline uint32_t ndim() const { + return ndim_; + } + /*! + * \brief get corresponding index + * \param i dimension index + * \return the corresponding dimension size + */ + inline ValueType& operator[](size_t i) { + return begin()[i]; + } + /*! + * \brief get corresponding index + * \param i dimension index + * \return the corresponding dimension size + */ + inline const ValueType& operator[](size_t i) const { + return begin()[i]; + } + /*! + * \brief Save Tuple to JSON. + * \param writer JSONWriter + */ + inline void Save(dmlc::JSONWriter* writer) const { + std::vector tmp(begin(), end()); + writer->Write(tmp); + } + /*! + * \brief Load Tuple from JSON. + * \param reader JSONReader + */ + inline void Load(dmlc::JSONReader* reader) { + std::vector tmp; + reader->Read(&tmp); + this->assign(tmp.begin(), tmp.end()); + } + /*! + * \brief allow output string of tuple to ostream + * \param os the output stream + * \param t the tuple + * \return the ostream + */ + friend std::ostream &operator<<(std::ostream &os, const Tuple &t) { + os << '['; + const ValueType* begin = t.begin(); + const ValueType* end = t.end(); + for (const ValueType* it = begin; it != end; ++it) { + if (it != begin) os << ','; + os << *it; + } + os << ']'; + return os; + } + /*! + * \brief read tuple from the istream + * \param is the input stream + * \param t The tuple + * \return the istream + */ + friend std::istream &operator>>(std::istream &is, Tuple &t) { + // get ( + while (true) { + char ch = is.peek(); + if (isdigit(ch) || ch == '-') { + ValueType idx; + if (is >> idx) { + t.assign(&idx, &idx + 1); + } + return is; + } + is.get(); + if (ch == '(' || ch == '[') break; + if (!isspace(ch)) { + is.setstate(std::ios::failbit); + return is; + } + } + // Handle empty tuple + while (isspace(is.peek())) { + is.get(); + } + if (is.peek() == ')' || is.peek() == ']') { + is.get(); + return is; + } + // Handle non-empty tuple + ValueType idx; + std::vector tmp; + while (is >> idx) { + tmp.push_back(idx); + char ch; + do { + ch = is.get(); + } while (isspace(ch)); + if (std::is_integral::value && ch == 'L') { + ch = is.get(); + } + if (ch == ',') { + while (true) { + ch = is.peek(); + if (isspace(ch)) { + is.get(); continue; + } + if (ch == ')' || ch == ']') { + is.get(); break; + } + break; + } + if (ch == ')' || ch == ']') break; + } else if (ch == ')' || ch == ']') { + break; + } else { + is.setstate(std::ios::failbit); + return is; + } + } + t.assign(tmp.begin(), tmp.end()); + return is; + } + /*! + * \brief save the content into binary stream + * \param strm the output stream + * \tparam DType data type that save to + * \tparam TStream any stream type that have write + */ + template + inline void Save(TStream *strm) const; + /*! + * \brief load the content from binary stream + * \param strm the output stream + * \tparam DType data type that load from + * \tparam TStream any stream type that have write + * \return whether the load is successful + */ + template + inline bool Load(TStream *strm); + + protected: + // stack cache size + static const uint32_t kStackCache = 4; + /*! \brief number of dimension of the tuple */ + uint32_t ndim_{0}; + /*! \brief number of cells allocated in data_heap_ */ + uint32_t num_heap_allocated_{0}; + /*! \brief in stack space used to store shape when it is small */ + ValueType data_stack_[kStackCache]; + /*! \brief space to store shape when dimension is big*/ + ValueType* data_heap_{nullptr}; + // internal function to change the dimension + inline void SetDim(uint32_t ndim) { + if (ndim > kStackCache && + ndim > num_heap_allocated_) { + delete [] data_heap_; + data_heap_ = new ValueType[ndim]; + num_heap_allocated_ = ndim; + } + ndim_ = ndim; + } +}; + +/*! + * \brief A Shape class that is used to represent shape of each tensor. + */ +class TShape : public Tuple { + public: + /*! \brief default constructor */ + TShape() = default; + /*! + * constructor to construct a shape with all 1. + * \param ndim the number of dimension + */ + inline TShape(uint32_t ndim) { // NOLINT(*) + this->SetDim(ndim); + std::fill_n(begin(), ndim, 1); + } + /*! + * \brief copy constructor of TShape + * \param s source shape. + */ + inline TShape(const Tuple& s) { // NOLINT(*) + this->assign(s.begin(), s.end()); + } + /*! + * \brief constructor from initializer list + * \param init the initializer_list + */ + inline TShape(std::initializer_list init) { + this->assign(init.begin(), init.end()); + } + /*! + * \brief move constructor. + * \param s source shape. + */ + inline TShape(Tuple&& s) { // NOLINT(*) + this->swap(s); + } + /*! + * \brief construct the Tuple from content of iterator + * \param begin the beginning of iterator + * \param end end the end of the iterator + * \tparam RandomAccessIterator iterator type + */ + template + inline TShape(RandomAccessIterator begin, + RandomAccessIterator end) { + this->assign(begin, end); + } + /*! + * \brief assignment function from tshape + * \param src source shape. + * \return self. + */ + inline TShape& operator=(const Tuple& src) { + this->assign(src.begin(), src.end()); + return *this; + } + /*! + * \brief move assignment function from tshape + * \param src source shape. + * \return self. + */ + inline TShape& operator=(Tuple&& src) { // NOLINT(*) + TShape(std::move(src)).swap(*this); // NOLINT(*) + return *this; + } + /*! \return total number of elements in the shape */ + inline size_t Size() const { + dim_t size = 1; + const dim_t* start = begin(), *fin = end(); + for (const dim_t* it = start; it != fin; ++it) { + size *= *it; + } + return size; + } + /*! + * \return product shape in [dimstart,dimend) + * \param dimstart start dimension + * \param dimend end dimension + */ + inline size_t ProdShape(int dimstart, int dimend) const { + dim_t num = 1; + const dim_t *d = this->data(); + for (int i = dimstart; i < dimend; ++i) { + num *= d[i]; + } + return num; + } + /*! \return the begin data pointer to content of the tuple */ + inline const dim_t *data() const { + return begin(); + } + /*! \return the begin data pointer to content of the tuple */ + inline dim_t *data() { + return begin(); + } +#ifdef MSHADOW_XINLINE + template + inline TShape(const mshadow::Shape &s) {// NOLINT(*) + this->assign(s.shape_, s.shape_ + dim); + } + + template + inline TShape(mshadow::Shape &&s) {// NOLINT(*) + this->assign(s.shape_, s.shape_ + dim); + } + /*! + * \brief assignment from shape + * \param shape source shape + * \tparam dim shape dimension + * \return reference of self + */ + template + inline TShape &operator=(const mshadow::Shape &shape) { + this->assign(shape.shape_, shape.shape_ + dim); + return *this; + } + /*! + * \brief get the shape of tensor specifying dim + * \return the shape requested + * \tparam dim dimension of the tensor + */ + template + inline mshadow::Shape get() const { + CHECK_EQ(dim, static_cast(ndim())) + << "dimension do not match target dimension " << dim << " vs " << ndim(); + const dim_t *d = this->data(); + mshadow::Shape s; + for (int i = 0; i < dim; ++i) { + s[i] = d[i]; + } + return s; + } + /*! + * flatten the higher dimension to second dimension, return a 2D shape + * \return the flat 2d shape + */ + inline mshadow::Shape<2> FlatTo2D(void) const { + mshadow::Shape<2> s; + if (ndim() == 0) return mshadow::Shape2(0, 0); + const dim_t *d = this->data(); + s.shape_[1] = d[ndim() - 1]; + dim_t ymax = 1; + for (size_t i = 1; i < ndim(); ++i) { + ymax *= d[i - 1]; + } + s.shape_[0] = ymax; + return s; + } + /*! + * flatten the shape into three parts: [0, axis_begin), [axis_begin, axis_end], (axis_end, ndim) + * \param axis_begin The beginning axis specified. + * \param axis_end The ending axis specified. + * \return the flat 3d shape + */ + inline mshadow::Shape<3> FlatTo3D(size_t axis_begin, size_t axis_end) const { + CHECK(axis_end >= axis_begin); + mshadow::Shape<3> s; + if (ndim() == 0) return mshadow::Shape3(0, 0, 0); + const dim_t *d = this->data(); + s.shape_[0] = 1; + s.shape_[1] = 1; + s.shape_[2] = 1; + + for (size_t i = 0; i < axis_begin; ++i) { + s.shape_[0] *= d[i]; + } + for (size_t i = axis_begin; i <= axis_end; ++i) { + s.shape_[1] *= d[i]; + } + for (size_t i = axis_end + 1; i < ndim(); ++i) { + s.shape_[2] *= d[i]; + } + return s; + } + /*! + * flatten the axis before and after the specified axis, so it becomes 3D tensor + * \param axis The axis specified. + * \return the flat 3d shape + */ + inline mshadow::Shape<3> FlatTo3D(size_t axis) const { + return FlatTo3D(axis, axis); + } + inline bool operator==(const TShape &s) const { + if (ndim() != s.ndim()) return false; + return std::equal(begin(), end(), s.begin()); + } + inline bool operator!=(const TShape &s) const { + return !(*this == s); + } + /*! + * \return whether two shape equals + * \param s the shape to compare against + * \tparam dim dimension of the shape + */ + template + inline bool operator==(const mshadow::Shape &s) const { + if (ndim_ != dim) return false; + const dim_t *d = dim <= kStackCache ? data_stack_ : data_heap_; + for (size_t i = 0; i < dim; ++i) { + if (d[i] != s.shape_[i]) return false; + } + return true; + } + /*! + * \return whether two shape not equals + * \param s the shape to compare against + * \tparam dim dimension of the shape + */ + template + inline bool operator!=(const mshadow::Shape &s) const { + return !(*this == s); + } +#endif +}; + +/*! \brief helper function to cast type of container elements */ +template +inline DstIter ShapeTypeCast(const SrcIter begin, + const SrcIter end, + DstIter dst_begin) { + typedef typename std::iterator_traits::value_type SrcDType; + typedef typename std::iterator_traits::value_type DstDType; + auto cast = [](const SrcDType& dim) { return static_cast(dim); }; + return std::transform(begin, end, dst_begin, cast); +} + +/*! \brief helper function to transform a container to TShape with type cast */ +template +inline TShape ShapeTypeCast(const SrcIter begin, const SrcIter end) { + size_t ndim = std::distance(begin, end); + TShape res(ndim); + ShapeTypeCast(begin, end, res.begin()); + return res; +} + +/*! \tparam ValueType The type of data stored inside tuple. */ +template +template +inline void Tuple::Save(TStream *strm) const { + strm->Write(&ndim_, sizeof(ndim_)); + if (typeid(DType) == typeid(ValueType)) { + strm->Write(begin(), sizeof(ValueType) * ndim_); + } else { + std::vector buffer(ndim_); + ShapeTypeCast(begin(), end(), buffer.data()); + strm->Write(buffer.data(), sizeof(DType) * ndim_); + } +} + +/*! \tparam ValueType The type of data stored inside tuple. */ +template +template +inline bool Tuple::Load(TStream *strm) { + if (strm->Read(&ndim_, sizeof(ndim_)) != sizeof(ndim_)) return false; + this->SetDim(ndim_); + size_t nread = sizeof(DType) * ndim_; + if (typeid(DType) == typeid(ValueType)) { + if (strm->Read(begin(), nread) != nread) return false; + } else { + std::vector buffer(ndim_); + if (strm->Read(buffer.data(), nread) != nread) return false; + ShapeTypeCast(buffer.begin(), buffer.end(), begin()); + } + return true; +} + +} // namespace mxnet + +namespace std { +/*! \brief hash function for Tuple. */ +template +struct hash > { + /*! \brief hash a Tuple into unsigned int */ + size_t operator()(const mxnet::Tuple& val) const { + std::hash hash_uint; + size_t res = hash_uint(val.ndim()); + for (uint32_t i = 0; i < val.ndim(); ++i) { + res = dmlc::HashCombine(res, val[i]); + } + return res; + } +}; + +/*! \brief hash function for TShape. */ +template<> +struct hash { + /*! \brief hash a TShape into unsigned int */ + size_t operator()(const mxnet::TShape& val) const { + std::hash hash_uint; + size_t res = hash_uint(val.ndim()); + for (uint32_t i = 0; i < val.ndim(); ++i) { + res = dmlc::HashCombine(res, val[i]); + } + return res; + } +}; +} // namespace std + +namespace dmlc { +/*! \brief description for optional TShape */ +DMLC_DECLARE_TYPE_NAME(optional, "Shape or None"); +// avoid low version of MSVC +#if !defined(_MSC_VER) +template +struct type_name_helper > { + static inline std::string value() { + return "tuple of <" + type_name() + ">"; + } +}; +#endif +} // namespace dmlc + +namespace mxnet { +/*! + * \brief The result holder of shape of each NodeEntry in the graph. + * \note Stored under graph.attrs["shape"], provided by Pass "InferShape" + * + * \code + * Graph g = ApplyPass(src_graph, "InferShape"); + * const ShapeVector& shapes = g.GetAttr("shape"); + * // get shape by entry id + * TShape entry_shape = shapes[g.indexed_graph().entry_id(my_entry)]; + * \endcode + * + * \sa FInferShape + */ +using ShapeVector = std::vector; + +/*! + * \brief Shape inference function. + * Update the shapes given the input shape information. + * TShape.ndim() == -1 means the shape is still unknown. + * + * \note Register under "FInferShape", + * by default do not update any shapes. + * + * FInferShape is needed by shape inference + */ +using FInferShape = nnvm::FInferNodeEntryAttr; + +} // namespace mxnet + +#endif // MXNET_TUPLE_H_ diff --git a/julia/LICENSE.md b/julia/LICENSE.md index 5ecf95ac60bc..3e2c5a2673b8 100644 --- a/julia/LICENSE.md +++ b/julia/LICENSE.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + The MXNet.jl package is licensed under version 2.0 of the Apache License: > Copyright (c) 2015-2018: diff --git a/julia/NEWS.md b/julia/NEWS.md index 3da119496fac..4a6c1a210e17 100644 --- a/julia/NEWS.md +++ b/julia/NEWS.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # v1.5.0 (#TBD) * Following material from `mx` module got exported (#TBD): diff --git a/julia/README-DEV.md b/julia/README-DEV.md index a1d6fa9012fc..25a6c6d93ab9 100644 --- a/julia/README-DEV.md +++ b/julia/README-DEV.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Workflow for making a release 1. Update `NEWS.md` to list important changes diff --git a/julia/README.md b/julia/README.md index 91a3981464be..43b8deb0666a 100644 --- a/julia/README.md +++ b/julia/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet [![MXNet](http://pkg.julialang.org/badges/MXNet_0.6.svg)](http://pkg.julialang.org/?pkg=MXNet) diff --git a/julia/docs/src/api.md b/julia/docs/src/api.md index 4984129863d0..60cb0831d1bf 100644 --- a/julia/docs/src/api.md +++ b/julia/docs/src/api.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # API Documentation ```@contents diff --git a/julia/docs/src/api/callback.md b/julia/docs/src/api/callback.md index f67811cc41fe..5a35e5047120 100644 --- a/julia/docs/src/api/callback.md +++ b/julia/docs/src/api/callback.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Callback in training ```@autodocs diff --git a/julia/docs/src/api/context.md b/julia/docs/src/api/context.md index 93ccf83e51ba..2daabe2db41b 100644 --- a/julia/docs/src/api/context.md +++ b/julia/docs/src/api/context.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Context ```@autodocs diff --git a/julia/docs/src/api/executor.md b/julia/docs/src/api/executor.md index b560c7a0864d..c3037dfff60b 100644 --- a/julia/docs/src/api/executor.md +++ b/julia/docs/src/api/executor.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Executor ```@autodocs diff --git a/julia/docs/src/api/initializer.md b/julia/docs/src/api/initializer.md index d0aad2def4cd..b2515263f93a 100644 --- a/julia/docs/src/api/initializer.md +++ b/julia/docs/src/api/initializer.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Initializer ```@autodocs diff --git a/julia/docs/src/api/io.md b/julia/docs/src/api/io.md index 7312259dbf3c..34ad3c42bce7 100644 --- a/julia/docs/src/api/io.md +++ b/julia/docs/src/api/io.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Data Providers Data providers are wrappers that load external data, be it images, text, or general tensors, diff --git a/julia/docs/src/api/kvstore.md b/julia/docs/src/api/kvstore.md index 34a5027f85fb..e6bf852b2f43 100644 --- a/julia/docs/src/api/kvstore.md +++ b/julia/docs/src/api/kvstore.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Key-Value Store ```@autodocs diff --git a/julia/docs/src/api/metric.md b/julia/docs/src/api/metric.md index 63cca0cc41ba..163f27a5b397 100644 --- a/julia/docs/src/api/metric.md +++ b/julia/docs/src/api/metric.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Evaluation Metrics Evaluation metrics provide a way to evaluate the performance of a learned model. diff --git a/julia/docs/src/api/model.md b/julia/docs/src/api/model.md index f793c7c406c7..63137532de6a 100644 --- a/julia/docs/src/api/model.md +++ b/julia/docs/src/api/model.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Model The model API provides convenient high-level interface to do training and predicting on diff --git a/julia/docs/src/api/ndarray.md b/julia/docs/src/api/ndarray.md index 5877d8257758..8cc4948e4dde 100644 --- a/julia/docs/src/api/ndarray.md +++ b/julia/docs/src/api/ndarray.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # NDArray API ## Arithmetic Operations diff --git a/julia/docs/src/api/nn-factory.md b/julia/docs/src/api/nn-factory.md index 833d9a3efd53..70ecfd2f0157 100644 --- a/julia/docs/src/api/nn-factory.md +++ b/julia/docs/src/api/nn-factory.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Neural Network Factory Neural network factory provide convenient helper functions to define diff --git a/julia/docs/src/api/symbolic-node.md b/julia/docs/src/api/symbolic-node.md index ef731d9f7d00..b4b1c0167e5e 100644 --- a/julia/docs/src/api/symbolic-node.md +++ b/julia/docs/src/api/symbolic-node.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Symbolic API ```@autodocs diff --git a/julia/docs/src/api/visualize.md b/julia/docs/src/api/visualize.md index 429a927012e4..e401a888cc81 100644 --- a/julia/docs/src/api/visualize.md +++ b/julia/docs/src/api/visualize.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Network Visualization ```@autodocs diff --git a/julia/docs/src/index.md b/julia/docs/src/index.md index b6a51fc162ad..11a77670eae3 100644 --- a/julia/docs/src/index.md +++ b/julia/docs/src/index.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Documentation [MXNet.jl](https://github.com/dmlc/MXNet.jl) is the diff --git a/julia/docs/src/tutorial/char-lstm.md b/julia/docs/src/tutorial/char-lstm.md index 369bcddd53e9..3961744f282d 100644 --- a/julia/docs/src/tutorial/char-lstm.md +++ b/julia/docs/src/tutorial/char-lstm.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Generating Random Sentence with LSTM RNN ======================================== diff --git a/julia/docs/src/tutorial/mnist.md b/julia/docs/src/tutorial/mnist.md index 916e46deb853..cc5267071f11 100644 --- a/julia/docs/src/tutorial/mnist.md +++ b/julia/docs/src/tutorial/mnist.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Digit Recognition on MNIST ========================== diff --git a/julia/docs/src/user-guide/faq.md b/julia/docs/src/user-guide/faq.md index 8fd8a6b34551..2799584f5472 100644 --- a/julia/docs/src/user-guide/faq.md +++ b/julia/docs/src/user-guide/faq.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + FAQ === diff --git a/julia/docs/src/user-guide/install.md b/julia/docs/src/user-guide/install.md index f1d5eeefacfe..d1a90a808507 100644 --- a/julia/docs/src/user-guide/install.md +++ b/julia/docs/src/user-guide/install.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Installation Guide ================== diff --git a/julia/docs/src/user-guide/overview.md b/julia/docs/src/user-guide/overview.md index 5815bc6d772c..d1d32e80ebd7 100644 --- a/julia/docs/src/user-guide/overview.md +++ b/julia/docs/src/user-guide/overview.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Overview ## MXNet.jl Namespace diff --git a/julia/examples/char-lstm/README.md b/julia/examples/char-lstm/README.md index ff16ee0a3ae9..ac745dd4cc41 100644 --- a/julia/examples/char-lstm/README.md +++ b/julia/examples/char-lstm/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # LSTM char-rnn Because we explicitly unroll the LSTM/RNN over time for a fixed sequence length, diff --git a/julia/plugins/README.md b/julia/plugins/README.md index 38882889f494..f9925cbdfbde 100644 --- a/julia/plugins/README.md +++ b/julia/plugins/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Plugins of MXNet.jl This directory contains *plugins* of MXNet.jl. A plugin is typically a component that could be part of MXNet.jl, but excluded from the `mx` namespace. The plugins are included here primarily for two reasons: diff --git a/julia/src/executor.jl b/julia/src/executor.jl index 29c21c8f481f..e565617976ce 100644 --- a/julia/src/executor.jl +++ b/julia/src/executor.jl @@ -169,7 +169,7 @@ function simple_bind(self::SymbolicNode, ctx::Context; end end - aux_arrays = [zeros(shape, ctx) for shape in aux_shapes] + aux_arrays = NDArray[zeros(shape, ctx) for shape in aux_shapes] return bind(self, ctx, arg_arrays, args_grad=grad_arrays, grad_req=grad_req, aux_states=aux_arrays) end diff --git a/julia/test/runtests.jl b/julia/test/runtests.jl index 4e5f273950a7..e75df67ad8a4 100644 --- a/julia/test/runtests.jl +++ b/julia/test/runtests.jl @@ -39,7 +39,7 @@ include(joinpath(@__DIR__, "common.jl")) test_dir(joinpath(@__DIR__, "unittest")) # run the basic MNIST mlp example - if haskey(ENV, "CONTINUOUS_INTEGRATION") + if haskey(ENV, "INTEGRATION_TEST") @testset "MNIST Test" begin include(joinpath(BASEDIR, "examples", "mnist", "mlp-test.jl")) end diff --git a/make/pip/pip_darwin_cpu.mk b/make/pip/pip_darwin_cpu.mk index bf240ad128cd..87a967933f9e 100644 --- a/make/pip/pip_darwin_cpu.mk +++ b/make/pip/pip_darwin_cpu.mk @@ -75,7 +75,7 @@ USE_CUDNN = 0 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 0 +ENABLE_CUDA_RTC = 0 # use openmp for parallelization USE_OPENMP = 0 diff --git a/make/pip/pip_darwin_mkl.mk b/make/pip/pip_darwin_mkl.mk index 30b328c6420e..339c95fdd502 100644 --- a/make/pip/pip_darwin_mkl.mk +++ b/make/pip/pip_darwin_mkl.mk @@ -75,7 +75,7 @@ USE_CUDNN = 0 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 0 +ENABLE_CUDA_RTC = 0 # use openmp for parallelization USE_OPENMP = 0 diff --git a/make/pip/pip_linux_cpu.mk b/make/pip/pip_linux_cpu.mk index 22bb32b4d1b8..d680f6da38d2 100644 --- a/make/pip/pip_linux_cpu.mk +++ b/make/pip/pip_linux_cpu.mk @@ -75,7 +75,7 @@ USE_CUDNN = 0 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 0 +ENABLE_CUDA_RTC = 0 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu100.mk b/make/pip/pip_linux_cu100.mk index 514603b94ab6..0f3f84a417a8 100644 --- a/make/pip/pip_linux_cu100.mk +++ b/make/pip/pip_linux_cu100.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu100mkl.mk b/make/pip/pip_linux_cu100mkl.mk index da4127f0dd07..b4792e3d81bb 100644 --- a/make/pip/pip_linux_cu100mkl.mk +++ b/make/pip/pip_linux_cu100mkl.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu75.mk b/make/pip/pip_linux_cu75.mk index 02468aa9613a..ff05ec80c68d 100644 --- a/make/pip/pip_linux_cu75.mk +++ b/make/pip/pip_linux_cu75.mk @@ -75,7 +75,7 @@ USE_CUDNN = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu75mkl.mk b/make/pip/pip_linux_cu75mkl.mk index 6895383d4fec..a8259205a5d7 100644 --- a/make/pip/pip_linux_cu75mkl.mk +++ b/make/pip/pip_linux_cu75mkl.mk @@ -75,7 +75,7 @@ USE_CUDNN = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu80.mk b/make/pip/pip_linux_cu80.mk index 2a2b88de73a1..7abf09f02eb6 100644 --- a/make/pip/pip_linux_cu80.mk +++ b/make/pip/pip_linux_cu80.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu80mkl.mk b/make/pip/pip_linux_cu80mkl.mk index 40cef4ed2110..be42cca0ef2b 100644 --- a/make/pip/pip_linux_cu80mkl.mk +++ b/make/pip/pip_linux_cu80mkl.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu90.mk b/make/pip/pip_linux_cu90.mk index 88e55180b738..12bc7741bef4 100644 --- a/make/pip/pip_linux_cu90.mk +++ b/make/pip/pip_linux_cu90.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu90mkl.mk b/make/pip/pip_linux_cu90mkl.mk index 9c4adf07bb6a..770f292598c3 100644 --- a/make/pip/pip_linux_cu90mkl.mk +++ b/make/pip/pip_linux_cu90mkl.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu91.mk b/make/pip/pip_linux_cu91.mk index ea62510396ef..9e399f97d18f 100644 --- a/make/pip/pip_linux_cu91.mk +++ b/make/pip/pip_linux_cu91.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu91mkl.mk b/make/pip/pip_linux_cu91mkl.mk index 232ec47979cc..80ef690a00e5 100644 --- a/make/pip/pip_linux_cu91mkl.mk +++ b/make/pip/pip_linux_cu91mkl.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu92.mk b/make/pip/pip_linux_cu92.mk index 01d02cfefba9..b0731addbc2d 100644 --- a/make/pip/pip_linux_cu92.mk +++ b/make/pip/pip_linux_cu92.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_cu92mkl.mk b/make/pip/pip_linux_cu92mkl.mk index 1abde3d71d7f..768fcc9c1cd0 100644 --- a/make/pip/pip_linux_cu92mkl.mk +++ b/make/pip/pip_linux_cu92mkl.mk @@ -78,7 +78,7 @@ USE_NCCL = 1 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 1 +ENABLE_CUDA_RTC = 1 # use openmp for parallelization USE_OPENMP = 1 diff --git a/make/pip/pip_linux_mkl.mk b/make/pip/pip_linux_mkl.mk index 96b88850c21a..ea11061e1e82 100644 --- a/make/pip/pip_linux_mkl.mk +++ b/make/pip/pip_linux_mkl.mk @@ -75,7 +75,7 @@ USE_CUDNN = 0 # CUDA_ARCH := # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) -USE_NVRTC = 0 +ENABLE_CUDA_RTC = 0 # use openmp for parallelization USE_OPENMP = 1 diff --git a/matlab/README.md b/matlab/README.md index 939b7011a4f2..d5ef5d09fc8d 100644 --- a/matlab/README.md +++ b/matlab/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MATLAB binding for MXNet ### How to use diff --git a/perl-package/AI-MXNet/Changes b/perl-package/AI-MXNet/Changes index 8bd43f3be205..e67fe39b49ab 100644 --- a/perl-package/AI-MXNet/Changes +++ b/perl-package/AI-MXNet/Changes @@ -1,5 +1,11 @@ Revision history for Perl extension AI::MXNet +1.4 Mon Feb 18 11:54:07 PST 2019 + - Two more gluon loss classes + - Visualization fixes + - Gluon rnn rework, including hybridization + - Exposed GPU memory info to perl level. + 1.33 Thu Oct 4 13:25:56 PDT 2018 - Added randn function. - Internal SELU function on C++ layer. diff --git a/perl-package/AI-MXNet/META.json b/perl-package/AI-MXNet/META.json index bbbea734ccf8..37c573c279f5 100644 --- a/perl-package/AI-MXNet/META.json +++ b/perl-package/AI-MXNet/META.json @@ -30,7 +30,7 @@ }, "runtime" : { "requires" : { - "AI::MXNetCAPI" : "1.33", + "AI::MXNetCAPI" : "1.4", "AI::NNVMCAPI" : "1.3", "Function::Parameters" : "1.0705", "Hash::Ordered" : "0.012", @@ -45,5 +45,5 @@ } }, "release_status" : "stable", - "version" : "1.33" + "version" : "1.4" } diff --git a/perl-package/AI-MXNet/META.yml b/perl-package/AI-MXNet/META.yml index 26e37b572600..692ca0307948 100644 --- a/perl-package/AI-MXNet/META.yml +++ b/perl-package/AI-MXNet/META.yml @@ -34,7 +34,7 @@ no_index: - t - inc requires: - AI::MXNetCAPI: '1.33' + AI::MXNetCAPI: '1.4' AI::NNVMCAPI: '1.3' Function::Parameters: '1.0705' Hash::Ordered: '0.012' @@ -42,4 +42,4 @@ requires: Mouse: v2.1.0 PDL: '2.007' PDL::CCS: '1.23.4' -version: '1.33' +version: '1.4' diff --git a/perl-package/AI-MXNet/Makefile.PL b/perl-package/AI-MXNet/Makefile.PL index 6d70b21344c2..19aba3fee4a5 100644 --- a/perl-package/AI-MXNet/Makefile.PL +++ b/perl-package/AI-MXNet/Makefile.PL @@ -36,7 +36,7 @@ my %WriteMakefileArgs = ( "LICENSE" => "apache_2_0", "NAME" => "AI::MXNet", "PREREQ_PM" => { - "AI::MXNetCAPI" => "1.33", + "AI::MXNetCAPI" => "1.4", "AI::NNVMCAPI" => "1.3", "Function::Parameters" => "1.0705", "Hash::Ordered" => "0.012", @@ -46,7 +46,7 @@ my %WriteMakefileArgs = ( "GraphViz" => "2.14" }, "TEST_REQUIRES" => {}, - "VERSION" => "1.33", + "VERSION" => "1.4", "test" => { "TESTS" => "t/*.t" } diff --git a/perl-package/AI-MXNet/README b/perl-package/AI-MXNet/README index f370db3804e9..4935b6384071 100644 --- a/perl-package/AI-MXNet/README +++ b/perl-package/AI-MXNet/README @@ -1,5 +1,5 @@ This archive contains the distribution AI-MXNet, -version 1.33: +version 1.4: Perl interface to MXNet machine learning library diff --git a/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md b/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md index 658a77530a92..9be9b8ad023e 100644 --- a/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md +++ b/perl-package/AI-MXNet/examples/gluon/style_transfer/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + This directory provides AI::MXNet Implementation of MSG-Net real time style transfer, https://arxiv.org/abs/1703.06953 ### Stylize Images Using Pre-trained MSG-Net diff --git a/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md b/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md index debad272206f..3eb1bab508e5 100644 --- a/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md +++ b/perl-package/AI-MXNet/examples/sparse/matrix_factorization/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Matrix Factorization w/ Sparse Embedding =========== The example demonstrates the basic usage of the SparseEmbedding operator in MXNet, adapted based on @leopd's recommender examples. diff --git a/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md b/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md index 9a481d69edf2..4a01da4254e6 100644 --- a/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md +++ b/perl-package/AI-MXNet/examples/sparse/wide_deep/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + ## Wide and Deep Learning The example demonstrates how to train [wide and deep model](https://arxiv.org/abs/1606.07792). The [Census Income Data Set](https://archive.ics.uci.edu/ml/datasets/Census+Income) that this example uses for training is hosted by the [UC Irvine Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/). Tricks of feature engineering are adapted from tensorflow's [wide and deep tutorial](https://github.com/tensorflow/models/tree/master/official/wide_deep). diff --git a/perl-package/AI-MXNet/lib/AI/MXNet.pm b/perl-package/AI-MXNet/lib/AI/MXNet.pm index 6a559a394a9f..80699b14311c 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet.pm @@ -51,7 +51,7 @@ use AI::MXNet::Gluon; use AI::MXNet::NDArray::Sparse; use AI::MXNet::Symbol::Sparse; use AI::MXNet::Engine; -our $VERSION = '1.33'; +our $VERSION = '1.4'; sub import { diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Context.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Context.pm index 826e7baf905b..7ae99be7b99e 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Context.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Context.pm @@ -190,6 +190,30 @@ method num_gpus() return scalar(check_call(AI::MXNetCAPI::GetGPUCount())); } +=head2 gpu_memory_info + + Query CUDA for the free and total bytes of GPU global memory. + + Parameters + ---------- + $device_id=0 : int, optional + The device id of the GPU device. + + Raises + ------ + Will raise an exception on any CUDA error. + + Returns + ------- + ($free, $total) : (int, int) + Free and total memory in bytes. +=cut + +method gpu_memory_info($device_id=0) +{ + return check_call(AI::MXNetCAPI::GetGPUMemoryInformation64($device_id)); +} + method current_ctx() { return $AI::MXNet::current_ctx; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/Loss.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/Loss.pm index 7dea68ffa16d..3eb62eb5a2ef 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/Loss.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/Loss.pm @@ -824,4 +824,175 @@ method hybrid_forward( __PACKAGE__->register('AI::MXNet::Gluon::Loss'); +package AI::MXNet::Gluon::PoissonNLLLoss; +use AI::MXNet::Gluon::Mouse; +extends 'AI::MXNet::Gluon::Loss'; +has 'from_logits' => (is => 'ro', isa => 'Bool', default => 1); +has 'compute_full' => (is => 'ro', isa => 'Bool', default => 0); + +=head1 NAME + + AI::MXNet::Gluon::PoissonNLLLoss +=cut + +=head1 DESCRIPTION + + For a target (Random Variable) in a Poisson distribution, the function calculates the Negative + Log likelihood loss. + PoissonNLLLoss measures the loss accrued from a poisson regression prediction made by the model. + + .. math:: + L = \text{pred} - \text{target} * \log(\text{pred}) +\log(\text{target!}) + + `pred`, `target` can have arbitrary shape as long as they have the same number of elements. + + Parameters + ---------- + from_logits : boolean, default True + indicating whether log(predicted) value has already been computed. If True, the loss is computed as + :math:`\exp(\text{pred}) - \text{target} * \text{pred}`, and if False, then loss is computed as + :math:`\text{pred} - \text{target} * \log(\text{pred}+\text{epsilon})`.The default value + weight : float or None + Global scalar weight for loss. + batch_axis : int, default 0 + The axis that represents mini-batch. + compute_full: boolean, default False + Indicates whether to add an approximation(Stirling factor) for the Factorial term in the formula for the loss. + The Stirling factor is: + :math:`\text{target} * \log(\text{target}) - \text{target} + 0.5 * \log(2 * \pi * \text{target})` + epsilon: float, default 1e-08 + This is to avoid calculating log(0) which is not defined. + + + Inputs: + - **pred**: Predicted value + - **target**: Random variable(count or number) which belongs to a Poisson distribution. + - **sample_weight**: element-wise weighting tensor. Must be broadcastable + to the same shape as pred. For example, if pred has shape (64, 10) + and you want to weigh each sample in the batch separately, + sample_weight should have shape (64, 1). + + Outputs: + - **loss**: Average loss (shape=(1,1)) of the loss tensor with shape (batch_size,). +=cut + +method hybrid_forward( + GluonClass $F, GluonInput $pred, GluonInput $target, + Maybe[GluonInput] $sample_weight=, Maybe[Num] $epsilon=1e-08 +) +{ + $target = __PACKAGE__->_reshape_like($F, $target, $pred); + my $loss; + if($self->from_logits) + { + $loss = $F->exp($pred) - $target * $pred; + } + else + { + $loss = $pred - $target * $F->log($pred + $epsilon); + if($self->compute_full) + { + my $stirling_factor = $target * $F->log($target) - $target + 0.5 * $F->log(2 * $target * 3.1415926); + $stirling_factor *= ($target > 1); + $loss += $stirling_factor; + } + $loss = __PACKAGE__->_apply_weighting($F, $loss, $self->weight, $sample_weight); + } + return $F->mean($loss); +} + +__PACKAGE__->register('AI::MXNet::Gluon::Loss'); + +package AI::MXNet::Gluon::CosineEmbeddingLoss; +use AI::MXNet::Gluon::Mouse; +extends 'AI::MXNet::Gluon::Loss'; +has 'margin' => (is => 'rw', isa => 'Num', default => 0); + +=head1 NAME + + AI::MXNet::Gluon::CosineEmbeddingLoss +=cut + +=head1 DESCRIPTION + + For a target label 1 or -1, vectors input1 and input2, the function computes the cosine distance + between the vectors. This can be interpreted as how similar/dissimilar two input vectors are. + + .. math:: + + L = \sum_i \begin{cases} 1 - {cos\_sim({input1}_i, {input2}_i)} & \text{ if } {label}_i = 1\\ + {cos\_sim({input1}_i, {input2}_i)} & \text{ if } {label}_i = -1 \end{cases}\\ + cos\_sim(input1, input2) = \frac{{input1}_i.{input2}_i}{||{input1}_i||.||{input2}_i||} + + `input1`, `input2` can have arbitrary shape as long as they have the same number of elements. + + Parameters + ---------- + weight : float or None + Global scalar weight for loss. + batch_axis : int, default 0 + The axis that represents mini-batch. + margin : float + Margin of separation between correct and incorrect pair. + + + Inputs: + - **input1**: a tensor with arbitrary shape + - **input2**: another tensor with same shape as pred to which input1 is + compared for similarity and loss calculation + - **label**: A 1-D tensor indicating for each pair input1 and input2, target label is 1 or -1 + - **sample_weight**: element-wise weighting tensor. Must be broadcastable + to the same shape as input1. For example, if input1 has shape (64, 10) + and you want to weigh each sample in the batch separately, + sample_weight should have shape (64, 1). + + Outputs: + - **loss**: The loss tensor with shape (batch_size,). +=cut + +method hybrid_forward( + GluonClass $F, GluonInput $input1, GluonInput $input2, GluonInput $label, Maybe[GluonInput] $sample_weight= +) +{ + $input1 = __PACKAGE__->_reshape_like($F, $input1, $input2); + $label = $label->reshape([-1, 1]); + my $cos_sim = $self->_cosine_similarity($F, $input1, $input2); + my $y_1 = $label == 1; + my $y_minus_1 = $label == -1; + my $cos_sim_a = (1 - $cos_sim) * $y_1; + + my $z_array; + if($F eq 'AI::MXNet::NDArray') + { + $z_array = $F->array([0]); + } + else + { + $z_array = $F->zeros([1, 1]); + } + my $cos_sim_b = $F->broadcast_maximum($z_array, $y_minus_1 * ($cos_sim - $self->margin), { axis=>1 }); + my $loss = $cos_sim_a + $cos_sim_b; + $loss = __PACKAGE__->_apply_weighting($F, $loss, $self->weight, $sample_weight); + return $loss; +} + +method _cosine_similarity($F, $x, $y, $axis=-1) +{ + my $x_norm = $F->norm($x, axis=>$axis)->reshape([-1, 1]); + my $y_norm = $F->norm($y, axis=>$axis)->reshape([-1, 1]); + my $x_dot_y = $F->sum($x*$y, axis=>$axis)->reshape([-1, 1]); + my $eps_arr; + if($F eq 'AI::MXNet::NDArray') + { + $eps_arr = $F->array([1e-12]); + } + else + { + $eps_arr = $F->full([1, 1], 1e-12); + } + return ($x_dot_y / $F->broadcast_maximum($x_norm * $y_norm, $eps_arr)); +} + +__PACKAGE__->register('AI::MXNet::Gluon::Loss'); + 1; \ No newline at end of file diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/RNN/Cell.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/RNN/Cell.pm index c14b792e77d7..89493c7b8bfb 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/RNN/Cell.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/RNN/Cell.pm @@ -57,6 +57,7 @@ method _get_begin_state(GluonClass $F, $begin_state, GluonInput $inputs, $batch_ return $begin_state; } + method _format_sequence($length, $inputs, $layout, $merge, $in_layout=) { assert( @@ -118,7 +119,7 @@ method _format_sequence($length, $inputs, $layout, $merge, $in_layout=) if($merge) { $inputs = [map { $F->expand_dims($_, axis => $axis) } @{ $inputs }]; - $inputs = $F->concat(@{ $inputs }, dim => $axis); + $inputs = $F->stack(@{ $inputs }, axis => $axis); $in_axis = $axis; } } @@ -129,6 +130,54 @@ method _format_sequence($length, $inputs, $layout, $merge, $in_layout=) return ($inputs, $axis, $F, $batch_size); } +method _mask_sequence_variable_length($F, $data, $length, $valid_length, $time_axis, $merge) +{ + assert(defined $valid_length); + if(not blessed $data) + { + $data = $F->stack(@$data, axis=>$time_axis); + } + my $outputs = $F->SequenceMask($data, { sequence_length=>$valid_length, use_sequence_length=>1, + axis=>$time_axis}); + if(not $merge) + { + $outputs = $F->split($outputs, { num_outputs=>$length, axis=>$time_axis, + squeeze_axis=>1}); + if(not ref $outputs eq 'ARRAY') + { + $outputs = [$outputs]; + } + } + return $outputs; +} + +method _reverse_sequences($sequences, $unroll_step, $valid_length=) +{ + my $F; + if($sequences->[0]->isa('AI::MXNet::Symbol')) + { + $F = 'AI::MXNet::Symbol'; + } + else + { + $F = 'AI::MXNet::NDArray'; + } + + my $reversed_sequences; + if(not defined $valid_length) + { + $reversed_sequences = [reverse(@$sequences)]; + } + else + { + $reversed_sequences = $F->SequenceReverse($F->stack(@$sequences, axis=>0), + {sequence_length=>$valid_length, + use_sequence_length=>1}); + $reversed_sequences = $F->split($reversed_sequences, {axis=>0, num_outputs=>$unroll_step, squeeze_axis=>1}); + } + return $reversed_sequences; +} + =head1 NAME AI::MXNet::Gluon::RNN::RecurrentCell @@ -280,21 +329,39 @@ method unroll( Maybe[GluonInput] $inputs, Maybe[GluonInput] :$begin_state=, Str :$layout='NTC', - Maybe[Bool] :$merge_outputs= + Maybe[Bool] :$merge_outputs=, + Maybe[Bool] :$valid_length= ) { $self->reset(); - my ($F, $batch_size); - ($inputs, undef, $F, $batch_size) = $self->_format_sequence($length, $inputs, $layout, 0); + my ($F, $batch_size, $axis); + ($inputs, $axis, $F, $batch_size) = $self->_format_sequence($length, $inputs, $layout, 0); $begin_state //= $self->_get_begin_state($F, $begin_state, $inputs, $batch_size); my $states = $begin_state; my $outputs = []; + my $all_states = []; for my $i (0..$length-1) { my $output; ($output, $states) = $self->($inputs->[$i], $states); push @$outputs, $output; + if(defined $valid_length) + { + push @$all_states, $states; + } + } + if(defined $valid_length) + { + $states = []; + for(zip(@$all_states)) + { + push @$states, $F->SequenceLast($F->stack(@$_, axis=>0), + sequence_length=>$valid_length, + use_sequence_length=>1, + axis=>0); + } + $outputs = $self->_mask_sequence_variable_length($F, $outputs, $length, $valid_length, $axis, 1); } ($outputs) = $self->_format_sequence($length, $outputs, $layout, $merge_outputs); return ($outputs, $states); @@ -304,8 +371,17 @@ method _get_activation(GluonClass $F, GluonInput $inputs, Activation $activation { if(not blessed $activation) { + my %act = map { $_ => 1 } qw(tanh relu sigmoid softsign); + if(exists $act{$activation}) + { + return $F->$activation($inputs, %kwargs) + } return $F->Activation($inputs, act_type=>$activation, %kwargs); } + elsif(ref($activation) =~ /LeakyReLU/) + { + return $F->LeakyReLU($inputs, act_type=>'leaky', slope => $activation->alpha, %kwargs); + } else { return $activation->($inputs, %kwargs); @@ -430,7 +506,7 @@ has [qw/ method python_constructor_arguments() { [qw/ - hidden_size activation + hidden_size activation i2h_weight_initializer h2h_weight_initializer i2h_bias_initializer h2h_bias_initializer input_size @@ -476,16 +552,17 @@ method hybrid_forward( { my $prefix = "t${\ $self->counter}_"; my $i2h = $F->FullyConnected( - $inputs, $i2h_weight, $i2h_bias, + data => $inputs, weight => $i2h_weight, bias => $i2h_bias, num_hidden => $self->hidden_size, name => "${prefix}i2h" ); my $h2h = $F->FullyConnected( - $states->[0], $h2h_weight, $h2h_bias, + data => $states->[0], weight => $h2h_weight, bias => $h2h_bias, num_hidden => $self->hidden_size, name => "${prefix}h2h" ); - my $output = $self->_get_activation($F, $i2h + $h2h, $self->activation, name => "${prefix}out"); + my $i2h_plus_h2h = $F->elemwise_add($i2h, $h2h, name => "${prefix}plus0"); + my $output = $self->_get_activation($F, $i2h_plus_h2h, $self->activation, name => "${prefix}out"); return ($output, [$output]); } @@ -555,6 +632,7 @@ method python_constructor_arguments() /]; } + sub BUILD { my $self = shift; @@ -606,14 +684,18 @@ method hybrid_forward( num_hidden => $self->hidden_size*4, name => "${prefix}h2h" ); - my $gates = $i2h + $h2h; + my $gates = $F->elemwise_add($i2h, $h2h, name => "${prefix}plus0"); my @slice_gates = @{ $F->SliceChannel($gates, num_outputs => 4, name => "${prefix}slice") }; my $in_gate = $F->Activation($slice_gates[0], act_type=>"sigmoid", name => "${prefix}i"); my $forget_gate = $F->Activation($slice_gates[1], act_type=>"sigmoid", name => "${prefix}f"); my $in_transform = $F->Activation($slice_gates[2], act_type=>"tanh", name => "${prefix}c"); my $out_gate = $F->Activation($slice_gates[3], act_type=>"sigmoid", name => "${prefix}o"); - my $next_c = $F->_plus($forget_gate * $states->[1], $in_gate * $in_transform, name => "${prefix}state"); - my $next_h = $F->_mul($out_gate, $F->Activation($next_c, act_type=>"tanh"), name => "${prefix}out"); + my $next_c = $F->_plus( + $F->elemwise_mul($forget_gate, $states->[1], name => "${prefix}mul0"), + $F->elemwise_mul($in_gate, $in_transform, name => "${prefix}mul1"), + name => "${prefix}state" + ); + my $next_h = $F->_mul($out_gate, $F->Activation($next_c, act_type=>"tanh", name => "${prefix}activation0"), name => "${prefix}out"); return ($next_h, [$next_h, $next_c]); } @@ -735,10 +817,29 @@ method hybrid_forward( my ($i2h_r, $i2h_z, $h2h_r, $h2h_z); ($i2h_r, $i2h_z, $i2h) = @{ $F->SliceChannel($i2h, num_outputs => 3, name => "${prefix}i2h_slice") }; ($h2h_r, $h2h_z, $h2h) = @{ $F->SliceChannel($h2h, num_outputs => 3, name => "${prefix}h2h_slice") }; - my $reset_gate = $F->Activation($i2h_r + $h2h_r, act_type=>"sigmoid", name => "${prefix}r_act"); - my $update_gate = $F->Activation($i2h_z + $h2h_z, act_type=>"sigmoid", name => "${prefix}z_act"); - my $next_h_tmp = $F->Activation($i2h + $reset_gate * $h2h, act_type => "tanh", name => "${prefix}h_act"); - my $next_h = $F->_plus((1 - $update_gate) * $next_h_tmp, $update_gate * $prev_state_h, name => "${prefix}out"); + my $reset_gate = $F->Activation($F->elemwise_add($i2h_r, $h2h_r, name => "${prefix}plus0"), act_type=>"sigmoid", name => "${prefix}r_act"); + my $update_gate = $F->Activation($F->elemwise_add($i2h_z, $h2h_z, name => "${prefix}plus1"), act_type=>"sigmoid", name => "${prefix}z_act"); + my $next_h_tmp = $F->Activation( + $F->elemwise_add( + $i2h, + $F->elemwise_mul( + $reset_gate, $h2h, name => "${prefix}mul0" + ), + name => "${prefix}plus2" + ), + act_type => "tanh", + name => "${prefix}h_act" + ); + my $ones = $F->ones_like($update_gate, name => "${prefix}ones_like0"); + my $next_h = $F->_plus( + $F->elemwise_mul( + $F->elemwise_sub($ones, $update_gate, name => "${prefix}minus0"), + $next_h_tmp, + name => "${prefix}mul1" + ), + $F->elemwise_mul($update_gate, $prev_state_h, name => "${prefix}mul2"), + name => "${prefix}out" + ); return ($next_h, [$next_h]); } diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/RNN/Layer.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/RNN/Layer.pm index 2b6e8a5bdae4..08212ab20f6d 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/RNN/Layer.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Gluon/RNN/Layer.pm @@ -21,7 +21,7 @@ package AI::MXNet::Gluon::RNN::Layer; use AI::MXNet::Function::Parameters; use AI::MXNet::Gluon::Mouse; use AI::MXNet::Base; -extends 'AI::MXNet::Gluon::Block'; +extends 'AI::MXNet::Gluon::HybridBlock'; has 'hidden_size' => (is => 'rw', isa => 'Int'); has 'num_layers' => (is => 'rw', isa => 'Int'); @@ -29,18 +29,19 @@ has 'layout' => (is => 'rw', isa => 'Str'); has 'dropout' => (is => 'rw', isa => 'Num'); has 'bidirectional' => (is => 'rw', isa => 'Bool'); has 'input_size' => (is => 'rw', isa => 'Int', default => 0); +has 'projection_size' => (is => 'rw', isa => 'Maybe[Int]'); +has [qw/lstm_state_clip_min + lstm_state_clip_max/] => (is => 'rw', isa => 'Maybe[Num]'); +has 'lstm_state_clip_nan' => (is => 'rw', isa => 'Bool', default => 0); has [qw/ i2h_weight_initializer h2h_weight_initializer i2h_bias_initializer h2h_bias_initializer + h2r_weight_initializer /] => (is => 'rw', isa => 'Maybe[Initializer]'); has 'mode' => (is => 'rw', isa => 'Str'); has [qw/dir gates - i2h_weight - h2h_weight - i2h_bias - h2h_bias unfused/] => (is => 'rw', init_arg => undef); method python_constructor_arguments() @@ -50,7 +51,8 @@ method python_constructor_arguments() dropout bidirectional input_size i2h_weight_initializer h2h_weight_initializer i2h_bias_initializer h2h_bias_initializer - mode + mode projection_size h2r_weight_initializer + lstm_state_clip_min lstm_state_clip_max lstm_state_clip_nan /]; } @@ -61,41 +63,76 @@ sub BUILD ($self->layout eq 'TNC' or $self->layout eq 'NTC'), "Invalid layout [${\ $self->layout }]; must be one of ['TNC' or 'NTC']" ); - $self->i2h_weight([]); - $self->h2h_weight([]); - $self->i2h_bias([]); - $self->h2h_bias([]); $self->dir($self->bidirectional ? 2 : 1); $self->gates({qw/rnn_relu 1 rnn_tanh 1 lstm 4 gru 3/}->{$self->mode}); my ($ng, $ni, $nh) = ($self->gates, $self->input_size, $self->hidden_size); - for my $i (0..$self->num_layers-1) + if(not $self->projection_size) { - for my $j ($self->dir == 2 ? ('l', 'r') : ('l')) + for my $i (0..$self->num_layers-1) { - push @{ $self->i2h_weight }, $self->params->get( - "$j${i}_i2h_weight", shape=>[$ng*$nh, $ni], - init => $self->i2h_weight_initializer, - allow_deferred_init => 1 - ); - push @{ $self->h2h_weight }, $self->params->get( - "$j${i}_h2h_weight", shape=>[$ng*$nh, $nh], - init => $self->h2h_weight_initializer, - allow_deferred_init => 1 - ); - push @{ $self->i2h_bias }, $self->params->get( - "$j${i}_i2h_bias", shape=>[$ng*$nh], - init => $self->i2h_bias_initializer, - allow_deferred_init => 1 - ); - push @{ $self->h2h_bias }, $self->params->get( - "$j${i}_h2h_bias", shape=>[$ng*$nh], - init => $self->h2h_bias_initializer, - allow_deferred_init => 1 - ); + for my $j ($self->dir == 2 ? ('l', 'r') : ('l')) + { + $self->_register_param( + "$j${i}_i2h_weight", [$ng*$nh, $ni], + $self->i2h_weight_initializer + ); + $self->_register_param( + "$j${i}_h2h_weight", [$ng*$nh, $nh], + $self->h2h_weight_initializer + ); + $self->_register_param( + "$j${i}_i2h_bias", [$ng*$nh], + $self->i2h_bias_initializer, + ); + $self->_register_param( + "$j${i}_h2h_bias", [$ng*$nh], + $self->h2h_bias_initializer, + ); + } + $ni = $nh * $self->dir; + } + } + else + { + my $np = $self->projection_size; + for my $i (0..$self->num_layers-1) + { + for my $j ($self->dir == 2 ? ('l', 'r') : ('l')) + { + $self->_register_param( + "$j${i}_i2h_weight", [$ng*$nh, $ni], + $self->i2h_weight_initializer + ); + $self->_register_param( + "$j${i}_h2h_weight", [$ng*$nh, $np], + $self->h2h_weight_initializer + ); + $self->_register_param( + "$j${i}_i2h_bias", [$ng*$nh], + $self->i2h_bias_initializer, + ); + $self->_register_param( + "$j${i}_h2h_bias", [$ng*$nh], + $self->h2h_bias_initializer, + ); + $self->_register_param( + "$j${i}_h2r_weight", [$np, $nh], + $self->h2r_weight_initializer, + ); + } + $ni = $np * $self->dir; } - $ni = $nh * $self->dir; } - $self->unfused($self->_unfuse()); +} + +method _register_param($name, $shape, $init) +{ + my $p = $self->params->get( + $name, shape=>$shape, init=>$init, + allow_deferred_init=>1 + ); + $self->$name($p); + return $p; } use overload '""' => sub { @@ -119,15 +156,55 @@ use overload '""' => sub { return $s; }; +method _collect_params_with_prefix(Str $prefix='') +{ + $prefix .= '.' if($prefix); + my $pattern = qr/(l|r)(\d+)_(i2h|h2h)_(weight|bias)$/; + my $convert_key = sub { my ($m, $bidirectional) = @_; + my ($d, $l, $g, $t) = @$m; + if($bidirectional) + { + return "_unfused.$l.${d}_cell.${g}_$t"; + } + else + { + return "_unfused.$l.${g}_$t"; + } + }; + my $bidirectional = 0; + my %params = %{ $self->_reg_params }; + for my $k (keys %params) + { + $k =~ $pattern; + $bidirectional = 1 if $1 and $1 eq 'r'; + } + my %ret; + for my $k (keys %params) + { + $k =~ $pattern; + $ret{ $prefix . $convert_key->([$1, $2, $3, $4], $bidirectional) } = $params{$k}; + } + my $iter = $self->_children->iterator; + while(my ($name, $child) = $iter->()) + { + %ret = (%ret, %{ $child->_collect_params_with_prefix("$prefix$name") }); + } + return \%ret; +} + method state_info($batch_size=0) { confess('NotImplementedError'); } -# Unfuses the fused RNN in to a stack of rnn cells. method _unfuse() { + assert((not $self->projection_size), "_unfuse does not support projection layer yet!"); + assert( + (not $self->lstm_state_clip_min and not $self->lstm_state_clip_max), + "_unfuse does not support state clipping yet!" + ); my $get_cell = { rnn_relu => sub { my %kwargs = @_; @@ -218,89 +295,105 @@ method begin_state( } use Data::Dumper; -method forward(GluonInput $inputs, Maybe[GluonInput] $states=) +method hybrid_forward(GluonClass $F, GluonInput $inputs, @args) { - my $batch_size = $inputs->shape->[index($self->layout, 'N')]; - my $skip_states = not defined $states; - if($skip_states) + my $states; + if(@args) { - $states = $self->begin_state($batch_size, ctx=>$inputs->context); + if(not defined $args[0] or ref $args[0]) + { + $states = shift(@args); + undef $states if(ref $states eq 'ARRAY' and not @$states); + } } - if(blessed $states and $states->isa('AI::MXNet::NDArray')) + use Data::Dumper; + + my $batch_size; + if($F eq 'AI::MXNet::NDArray') { - $states = [$states]; + $batch_size = $inputs->shape->[index($self->layout, 'N')]; } - for(zip($states, $self->state_info($batch_size))) { - my ($state, $info) = @$_; - if(Dumper($state->shape) ne Dumper($info->{shape})) + my $skip_states = not defined $states; + if($skip_states) + { + if($F eq 'AI::MXNet::NDArray') { - my @state_shape = @{ $state->shape }; - confess("Invalid recurrent state shape. Expecting @{$info->{shape}}, got @state_shape."); + $states = $self->begin_state($batch_size, ctx=>$inputs->context, dtype=>$inputs->dtype); } - } - if($self->input_size == 0) - { - for my $i (0..$self->dir-1) + else { - $self->i2h_weight->[$i]->shape([$self->gates*$self->hidden_size, $inputs->shape->[2]]); - $self->i2h_weight->[$i]->_finish_deferred_init(); + $states = $self->begin_state(0, func=>sub { return AI::MXNet::Symbol->zeros(@_) }); } } - my $out; - if($inputs->context->device_type eq 'gpu') + if(blessed $states and ($states->isa('AI::MXNet::NDArray') or $states->isa('AI::MXNet::Symbol'))) { - $out = $self->_forward_gpu($inputs, $states); + $states = [$states]; } - else + if($F eq 'AI::MXNet::NDArray') { - $out = $self->_forward_cpu($inputs, $states); + for(zip($states, $self->state_info($batch_size))) + { + my ($state, $info) = @$_; + if(Dumper($state->shape) ne Dumper($info->{shape})) + { + my @state_shape = @{ $state->shape }; + confess("Invalid recurrent state shape. Expecting @{$info->{shape}}, got @state_shape."); + } + } } - - # out is (output, state) + my $out = $self->_forward_kernel($F, $inputs, $states, @args); return $skip_states ? $out->[0] : $out; } -method _forward_cpu($inputs, $states) +method _forward_kernel($F, $inputs, $states, %kwargs) { - my $ns = @{ $states }; - my $axis = index($self->layout, 'T'); - $states = [map { @{$_} } @{ $states }]; - my $outputs; - ($outputs, $states) = $self->unfused->unroll( - $inputs->shape->[$axis], $inputs, begin_state => $states, - layout => $self->layout, merge_outputs => 1 - ); - my @new_states; - for my $i (0..$ns-1) + if($self->layout eq 'NTC') { - my @tmp; - for (my $j = $i; $j < @{ $states }; $j += $ns) + $inputs = $F->swapaxes($inputs, dim1=>0, dim2=>1); + } + my @params; + if(not defined $self->projection_size) + { + for my $t ('weight', 'bias') { - push @tmp, $states->[$j]; + for my $l (0..$self->num_layers-1) + { + for my $d ($self->dir == 2 ? ('l', 'r') : ('l')) + { + for my $g ('i2h', 'h2h') + { + push @params, $kwargs{"$d${l}_${g}_$t"}->reshape([-1]); + } + } + } } - my $state = AI::MXNet::NDArray->concat((map { $_->reshape([1, @{ $_->shape }]) } @tmp), dim => 0); - push @new_states, $state; } - return [$outputs, \@new_states]; -} - -method _forward_gpu($inputs, $states) -{ - if($self->layout eq 'NTC') + else { - $inputs = $inputs->swapaxes(dim1 => 0, dim2 => 1); + for my $t ('weight', 'bias') + { + for my $l (0..$self->num_layers-1) + { + for my $d ($self->dir == 2 ? ('l', 'r') : ('l')) + { + for my $g ('i2h', 'h2h', 'h2r') + { + push @params, $kwargs{"$d${l}_${g}_$t"}->reshape([-1]) + unless($g eq 'h2r' and $t eq 'bias'); + } + } + } + } } - my $ctx = $inputs->context; - my @params = map { $_->data($ctx)->reshape([-1]) } map { @{ $_ } } ( - $self->i2h_weight, $self->h2h_weight, - $self->i2h_bias, $self->h2h_bias - ); - my $params = AI::MXNet::NDArray->concat(@params, dim => 0); - my $rnn = AI::MXNet::NDArray->RNN( - $inputs, $params, @{ $states }, state_size => $self->hidden_size, + my $params = $F->_rnn_param_concat(@params, dim=>0); + my $rnn = $F->RNN( + $inputs, $params, @{ $states }, { state_size => $self->hidden_size, num_layers => $self->num_layers, bidirectional => $self->dir == 2 ? 1 : 0, - p => $self->dropout, state_outputs => 1, mode => $self->mode - ); + p => $self->dropout, state_outputs => 1, mode => $self->mode, + (defined $self->lstm_state_clip_min ? (lstm_state_clip_min=>$self->lstm_state_clip_min) : ()), + (defined $self->lstm_state_clip_max ? (lstm_state_clip_max=>$self->lstm_state_clip_max) : ()), + (defined $self->lstm_state_clip_nan ? (lstm_state_clip_nan=>$self->lstm_state_clip_nan) : ()) + }); my $outputs; my @rnn = @{$rnn}; if($self->mode eq 'lstm') @@ -318,7 +411,6 @@ method _forward_gpu($inputs, $states) return [$outputs, $states]; } - package AI::MXNet::Gluon::RNN::RNN; =head1 NAME @@ -552,7 +644,10 @@ method state_info(DimSize $batch_size=0) { return [ { - shape => [$self->num_layers * $self->dir, $batch_size, $self->hidden_size], + shape => [ + $self->num_layers * $self->dir, $batch_size, + defined $self->projection_size ? $self->projection_size : $self->hidden_size + ], __layout__ => 'LNC' }, { diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/IO.pm b/perl-package/AI-MXNet/lib/AI/MXNet/IO.pm index 297ceb8c0b24..19e7cfdb8fe3 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/IO.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/IO.pm @@ -642,6 +642,7 @@ extends 'AI::MXNet::DataIter'; mx->io->CSVIter Returns the CSV file iterator. mx->io->LibSVMIter Returns the LibSVM iterator which returns data with csr storage type. mx->io->ImageRecordIter Iterates on image RecordIO files + mx->io->ImageRecordInt8Iter Iterating on image RecordIO files mx->io->ImageRecordUInt8Iter Iterating on image RecordIO files mx->io->MNISTIter Iterating on the MNIST dataset. mx->recordio->MXRecordIO Reads/writes RecordIO data format, supporting sequential read and write. diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm index 0359cc3640d4..75c8b1e3dad1 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm @@ -191,6 +191,16 @@ method call(Str|AI::MXNet::InitDesc $desc, AI::MXNet::NDArray $arr) $self->$method($desc, $arr); $self->_verbose_print($desc, $1, $arr); } + elsif($desc =~ /min$/) + { + $self->_init_zero($desc, $arr); + $self->_verbose_print($desc, 'min', $arr); + } + elsif($desc =~ /max$/) + { + $self->_init_one($desc, $arr); + $self->_verbose_print($desc, 'max', $arr); + } else { $self->_init_default($desc, $arr) @@ -250,6 +260,14 @@ method _legacy_init(Str $name, AI::MXNet::NDArray $arr) { $self->_init_zero($name, $arr); } + elsif($name =~ /min$/) + { + $self->_init_zero($name, $arr); + } + elsif($name =~ /max$/) + { + $self->_init_one($name, $arr); + } else { $self->_init_default($name, $arr); diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm index 3a7b6bab2e2c..72f6cc772178 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm @@ -1226,6 +1226,9 @@ method concatenate(ArrayRef[AI::MXNet::NDArray] $arrays, Index :$axis=0, :$alway :$repeat=1 : number, optional The repeating time of all elements. E.g repeat=3, the element a will be repeated three times --> a, a, a. + :$infer_range=0 : Bool + When set to 1, infer stop position from start, step, repeat, and + output tensor size. :$ctx : Context, optional The context of the NDArray, defaultw to current default context. :$dtype : data type, optional @@ -1237,7 +1240,7 @@ method concatenate(ArrayRef[AI::MXNet::NDArray] $arrays, Index :$axis=0, :$alway The created NDArray =cut -method arange(Index :$start=0, Maybe[Index] :$stop=, Index :$step=1, Index :$repeat=1, +method arange(Index :$start=0, Maybe[Index] :$stop=, Index :$step=1, Index :$repeat=1, Bool :$infer_range=0, AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, Dtype :$dtype='float32') { return __PACKAGE__->_arange({ @@ -1246,6 +1249,7 @@ method arange(Index :$start=0, Maybe[Index] :$stop=, Index :$step=1, Index :$rep step => $step, repeat => $repeat, dtype => $dtype, + infer_range => $infer_range, ctx => "$ctx" }); } diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm index 57bfdf1d977c..04dd1cbfc441 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm @@ -1411,16 +1411,19 @@ method ones(Shape :$shape, Dtype :$dtype='float32', Maybe[Str] :$name=, Maybe[St Parameters ---------- - start : number + :$start=0 : number Start of interval. The interval includes this value. The default start value is 0. - stop : number, optional + :$stop= : number, optional End of interval. The interval does not include this value. - step : number, optional + :$step=1.0 : number, optional Spacing between values - repeat : int, optional + :$repeat=1 : int, optional "The repeating time of all elements. E.g repeat=3, the element a will be repeated three times --> a, a, a. - dtype : type, optional + :$infer_range=0 : Bool + When set to 1, infer stop position from start, step, repeat, and + output tensor size. + :$dtype='float32' : type, optional The value type of the NDArray, default to np.float32 Returns @@ -1429,11 +1432,12 @@ method ones(Shape :$shape, Dtype :$dtype='float32', Maybe[Str] :$name=, Maybe[St The created Symbol =cut -method arange(Index :$start=0, Index :$stop=, Num :$step=1.0, Index :$repeat=1, Maybe[Str] :$name=, Dtype :$dtype='float32') +method arange(Index :$start=0, Index :$stop=, Num :$step=1.0, Index :$repeat=1, Bool :$infer_range=0, Maybe[Str] :$name=, Dtype :$dtype='float32') { return __PACKAGE__->_arange({ start => $start, (defined $stop ? (stop => $stop) : ()), - step => $step, repeat => $repeat, name => $name, dtype => $dtype + step => $step, repeat => $repeat, name => $name, dtype => $dtype, + infer_range => $infer_range }); } diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm index 20811f10fedf..1574ea58307f 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm @@ -172,6 +172,10 @@ method print_summary( $cur_param = $num_filter * 2; } } + elsif($op eq 'Embedding') + { + $cur_param = $node->{attrs}{input_dim} * $node->{attrs}{output_dim}; + } my $first_connection; if(not $pre_node) { diff --git a/perl-package/AI-MXNet/t/test_gluon_rnn.t b/perl-package/AI-MXNet/t/test_gluon_rnn.t index 83b294d110ce..51e6ad53e171 100644 --- a/perl-package/AI-MXNet/t/test_gluon_rnn.t +++ b/perl-package/AI-MXNet/t/test_gluon_rnn.t @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + use strict; use warnings; use Test::More tests => 77; @@ -276,14 +277,15 @@ sub check_rnn_layer_forward $inputs->attach_grad; my $out; mx->autograd->record(sub { - $out = $layer->($inputs, $states); if(defined $states) { + $out = $layer->($inputs, $states); ok(@$out == 2); $out = $out->[0]; } else { + $out = $layer->($inputs); ok(blessed $out and $out->isa('AI::MXNet::NDArray')); } $out->backward(); @@ -292,21 +294,19 @@ sub check_rnn_layer_forward my $pdl_out = $out->aspdl; my $pdl_dx = $inputs->grad->aspdl; $layer->hybridize; - mx->autograd->record(sub { - $out = $layer->($inputs, $states); if(defined $states) { - ok(@$out == 2); - $out = $out->[0] + ($out, $states) = $layer->($inputs, $states); + ok(blessed $out and $out->isa('AI::MXNet::NDArray')); } else { + $out = $layer->($inputs, $states); ok(blessed $out and $out->isa('AI::MXNet::NDArray')); } $out->backward(); }); - ok(almost_equal($pdl_out, $out->aspdl, 1e-3)); ok(almost_equal($pdl_dx, $inputs->grad->aspdl, 1e-3)); } @@ -314,21 +314,12 @@ sub check_rnn_layer_forward sub test_rnn_layers { check_rnn_layer_forward(gluon->rnn->RNN(10, 2), mx->nd->ones([8, 3, 20])); - check_rnn_layer_forward(gluon->rnn->RNN(10, 2), mx->nd->ones([8, 3, 20]), mx->nd->ones([2, 3, 10])); + check_rnn_layer_forward(gluon->rnn->RNN(10, 2, bidirectional=>1), mx->nd->ones([8, 3, 20]), mx->nd->ones([4, 3, 10])); check_rnn_layer_forward(gluon->rnn->LSTM(10, 2), mx->nd->ones([8, 3, 20])); - check_rnn_layer_forward(gluon->rnn->LSTM(10, 2), mx->nd->ones([8, 3, 20]), [mx->nd->ones([2, 3, 10]), mx->nd->ones([2, 3, 10])]); + check_rnn_layer_forward(gluon->rnn->LSTM(10, 2, bidirectional=>1), mx->nd->ones([8, 3, 20]), [mx->nd->ones([4, 3, 10]), mx->nd->ones([4, 3, 10])]); check_rnn_layer_forward(gluon->rnn->GRU(10, 2), mx->nd->ones([8, 3, 20])); - check_rnn_layer_forward(gluon->rnn->GRU(10, 2), mx->nd->ones([8, 3, 20]), mx->nd->ones([2, 3, 10])); - -# my $net = gluon->nn->Sequential(); -# $net->add(gluon->rnn->LSTM(10, 2, bidirectional=>1)); -# $net->add(gluon->nn->BatchNorm(axis=>2)); -# $net->add(gluon->nn->Flatten()); -# $net->add(gluon->nn->Dense(3, activation=>'relu')); -# $net->collect_params()->initialize(); -# mx->autograd->record(sub { -# $net->(mx->nd->ones([2, 3, 10]))->backward(); -# }); + check_rnn_layer_forward(gluon->rnn->GRU(10, 2, bidirectional=>1), mx->nd->ones([8, 3, 20]), mx->nd->ones([4, 3, 10])); } test_rnn_layers(); + diff --git a/perl-package/AI-MXNet/t/test_loss.t b/perl-package/AI-MXNet/t/test_loss.t index 7fc7ee81d0de..5a9e413bbfaf 100644 --- a/perl-package/AI-MXNet/t/test_loss.t +++ b/perl-package/AI-MXNet/t/test_loss.t @@ -17,7 +17,7 @@ use strict; use warnings; -use Test::More tests => 30; +use Test::More tests => 32; use AI::MXNet 'mx'; use AI::MXNet::Gluon 'gluon'; use AI::MXNet::TestUtils 'almost_equal'; @@ -435,3 +435,47 @@ sub test_triplet_loss test_triplet_loss(); +sub test_cosine_loss +{ + my $input1 = mx->nd->random->randn(3, 2); + my $input2 = mx->nd->random->randn(3, 2); + my $label = mx->nd->sign(mx->nd->random->randn($input1->shape->[0])); + + my $Loss = gluon->loss->CosineEmbeddingLoss(); + my $loss = $Loss->($input1, $input2, $label); + + my $numerator = mx->nd->sum($input1 * $input2, keepdims => 1, axis => 1); + my $denominator = mx->nd->sqrt(mx->nd->sum($input1**2, axis=>1, keepdims=>1)) + * + mx->nd->sqrt(mx->nd->sum($input2**2, axis=>1, keepdims=>1)); + my $pdl_loss = mx->nd->where( + ($label == 1), 1-$numerator/$denominator, + mx->nd->broadcast_maximum(mx->nd->array([0]), $numerator/$denominator, { axis=>1 }) + ); + ok(almost_equal($loss->aspdl, $pdl_loss->aspdl)); +} + +test_cosine_loss(); + +sub test_poisson_nllloss +{ + my $N = 1000; + mx->random->seed(1234); + srand(1234); + my $data = mx->random->poisson(shape=>[$N, 2]); + my $label = mx->random->poisson(lam=>4, shape=>[$N, 1]); + my $data_iter = mx->io->NDArrayIter($data, $label, batch_size=>20, label_name=>'label', shuffle=>1); + my $output = mx->sym->exp(get_net(1)); + my $l = mx->symbol->Variable('label'); + my $Loss = gluon->loss->PoissonNLLLoss(from_logits=>0); + my $loss = $Loss->($output, $l); + $loss = mx->sym->make_loss($loss); + my $mod = mx->mod->Module($loss, data_names=>['data'], label_names=>['label']); + local($AI::MXNet::Logging::silent) = 1; + $mod->fit($data_iter, num_epoch=>20, optimizer_params=>{learning_rate => 0.01}, + initializer=>mx->init->Normal(sigma=>0.1), eval_metric=>mx->metric->Loss(), + optimizer=>'adam'); + ok($mod->score($data_iter, mx->metric->Loss())->{loss} < 0.05); +} + +test_poisson_nllloss; diff --git a/perl-package/AI-MXNetCAPI/Changes b/perl-package/AI-MXNetCAPI/Changes index 08ad085abce9..cdbbdab57cdf 100644 --- a/perl-package/AI-MXNetCAPI/Changes +++ b/perl-package/AI-MXNetCAPI/Changes @@ -1,5 +1,8 @@ Revision history for Perl extension AI::MXNetCAPI +1.4 Mon Feb 18 11:54:07 PST 2019 + - Support for 64bit integers + 1.33 Thu Oct 4 13:25:56 PDT 2018 - Gluon: Better sparse support for KVStore. - Gpu memory info via mxnet api call. diff --git a/perl-package/AI-MXNetCAPI/META.json b/perl-package/AI-MXNetCAPI/META.json index 1849e6b3bc18..82bee1ace8f8 100644 --- a/perl-package/AI-MXNetCAPI/META.json +++ b/perl-package/AI-MXNetCAPI/META.json @@ -37,5 +37,5 @@ } }, "release_status" : "stable", - "version" : "1.33" + "version" : "1.4" } diff --git a/perl-package/AI-MXNetCAPI/META.yml b/perl-package/AI-MXNetCAPI/META.yml index d870f05fbe52..bd4af4047378 100644 --- a/perl-package/AI-MXNetCAPI/META.yml +++ b/perl-package/AI-MXNetCAPI/META.yml @@ -36,4 +36,4 @@ no_index: - inc requires: Test::More: '0' -version: '1.33' +version: '1.4' diff --git a/perl-package/AI-MXNetCAPI/README b/perl-package/AI-MXNetCAPI/README index 67b77ccd1614..848b4d03ab21 100644 --- a/perl-package/AI-MXNetCAPI/README +++ b/perl-package/AI-MXNetCAPI/README @@ -1,4 +1,4 @@ -AI-MXNetCAPI version 1.33 +AI-MXNetCAPI version 1.4 ===================== Swig interface to MXNet c api. diff --git a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm index bc7676047d76..e3b71f8efc92 100644 --- a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm +++ b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm @@ -18,7 +18,7 @@ package AI::MXNetCAPI; use base qw(DynaLoader); bootstrap AI::MXNetCAPI; -our $VERSION = '1.33'; +our $VERSION = '1.4'; 1; __END__ diff --git a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i index 68e11ca74e1a..50296c2aaba5 100644 --- a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i +++ b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i @@ -115,7 +115,7 @@ } } -%typemap(in,numinputs=0) (int *out) (int temp), (bool *out) (bool temp) +%typemap(in,numinputs=0) (int *out) (int temp), (bool *out) (bool temp), (uint64_t *out) (uint64_t temp) { temp = 0; $1 = &temp; @@ -131,6 +131,17 @@ } } +%typemap(argout) (uint64_t *out) +{ + if(!result) + { + $result = newSVnv((double)(*$1)); + sv_2mortal($result); + argvi++; + } +} + + %typemap(in,numinputs=0) (const int **out_stypes) (int* temp) { temp = NULL; diff --git a/perl-package/README.md b/perl-package/README.md index 93e34d1af37a..893a11d19f88 100644 --- a/perl-package/README.md +++ b/perl-package/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + [Perl API](https://mxnet.incubator.apache.org/api/perl/index.html) [![GitHub license](http://dmlc.github.io/img/apache2.svg)](../LICENSE) diff --git a/plugin/caffe/README.md b/plugin/caffe/README.md index 466305cc9b88..a497176a996f 100644 --- a/plugin/caffe/README.md +++ b/plugin/caffe/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # How to use Caffe operator in MXNet [Caffe](http://caffe.berkeleyvision.org/) has been a well-known and widely-used deep learning framework. Now MXNet has supported calling most caffe operators(layers) and loss functions directly in its symbolic graph! Using one's own customized caffe layer is also effortless. diff --git a/plugin/caffe/caffe_blob.cc b/plugin/caffe/caffe_blob.cc index 4d655f32dd01..6a75439f3e4e 100644 --- a/plugin/caffe/caffe_blob.cc +++ b/plugin/caffe/caffe_blob.cc @@ -72,7 +72,7 @@ void SetDataGradToBlob(caffeMemoryTypes memType, MXCAFFEBLOB(*blob, double)->set_gpu_diff(data_ptr); } -TShape Vector2TShape(const std::vector &vec_int) { +mxnet::TShape Vector2TShape(const std::vector &vec_int) { std::vector vec; for (uint32_t i = 0; i < vec_int.size(); ++i) vec.push_back(vec_int[i]); @@ -82,7 +82,7 @@ TShape Vector2TShape(const std::vector &vec_int) { return {vec.begin(), vec.end()}; } -std::vector TShape2Vector(const TShape &tshape) { +std::vector TShape2Vector(const mxnet::TShape &tshape) { std::vector s; for (uint32_t i =0 ; i < tshape.ndim(); ++i) s.push_back(tshape[i]); diff --git a/plugin/caffe/caffe_blob.h b/plugin/caffe/caffe_blob.h index a54c5c81ff47..6243b5dc8c88 100644 --- a/plugin/caffe/caffe_blob.h +++ b/plugin/caffe/caffe_blob.h @@ -39,8 +39,8 @@ namespace caffe { // Declare Memory Type for Caffe blob enum caffeMemoryTypes {Data, Grad, Non}; -TShape Vector2TShape(const std::vector &vec_int); -std::vector TShape2Vector(const TShape &tshape); +mxnet::TShape Vector2TShape(const std::vector &vec_int); +std::vector TShape2Vector(const mxnet::TShape &tshape); // implementation of tensor to blob, called by TensorToBlob template diff --git a/plugin/caffe/caffe_loss-inl.h b/plugin/caffe/caffe_loss-inl.h index 60b03b1d923b..98c714612dca 100644 --- a/plugin/caffe/caffe_loss-inl.h +++ b/plugin/caffe/caffe_loss-inl.h @@ -222,9 +222,9 @@ class CaffeLossProp : public OperatorProperty { } /*brief Set up caffeop to infer output shape*/ - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { + bool InferShape(mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape, + mxnet::ShapeVector *aux_shape) const override { using namespace mshadow; using ::caffe::Blob; using std::vector; @@ -236,7 +236,7 @@ class CaffeLossProp : public OperatorProperty { vector *> bot_blobs, top_blobs; for (int i = 0; i < param_.num_data; ++i) { - TShape tshape = (*in_shape)[i]; + mxnet::TShape tshape = (*in_shape)[i]; if (tshape.ndim() == 0) return false; auto blob_ptr = new Blob(); blob_ptr->Reshape(caffe::TShape2Vector(tshape)); @@ -251,7 +251,7 @@ class CaffeLossProp : public OperatorProperty { // Initialize out shapes out_shape->clear(); for (auto blob : top_blobs) { - TShape tshape = caffe::Vector2TShape(blob->shape()); + mxnet::TShape tshape = caffe::Vector2TShape(blob->shape()); out_shape->push_back(tshape); } @@ -288,7 +288,7 @@ class CaffeLossProp : public OperatorProperty { return NULL; } - Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + Operator* CreateOperatorEx(Context ctx, mxnet::ShapeVector *in_shape, std::vector *in_type) const override; diff --git a/plugin/caffe/caffe_loss.cc b/plugin/caffe/caffe_loss.cc index 5ce8bb247e86..47424d1cad80 100644 --- a/plugin/caffe/caffe_loss.cc +++ b/plugin/caffe/caffe_loss.cc @@ -47,10 +47,10 @@ Operator *CreateOp(CaffeLossParam param, int dtype) { } // DO_BIND_DISPATCH comes from static_operator_common.h -Operator *CaffeLossProp::CreateOperatorEx(Context ctx, std::vector *in_shape, +Operator *CaffeLossProp::CreateOperatorEx(Context ctx, mxnet::ShapeVector *in_shape, std::vector *in_type) const { std::vector out_type, aux_type; - std::vector out_shape, aux_shape; + mxnet::ShapeVector out_shape, aux_shape; out_type.resize(this->ListOutputs().size()); out_shape.resize(this->ListOutputs().size()); aux_type.resize(this->ListAuxiliaryStates().size()); diff --git a/plugin/caffe/caffe_op-inl.h b/plugin/caffe/caffe_op-inl.h index 2c1c9bac170a..b4ab0926199c 100644 --- a/plugin/caffe/caffe_op-inl.h +++ b/plugin/caffe/caffe_op-inl.h @@ -274,9 +274,9 @@ class CaffeOpProp : public OperatorProperty { * \brief Set up caffeOp_ to infer weights & output shape * \brief Initialize param_'s in & out dims */ - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { + bool InferShape(mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape, + mxnet::ShapeVector *aux_shape) const override { if (caffeOp_ == NULL) caffeOp_ = caffe::LayerRegistry::CreateLayer(param_.prototxt); using namespace mshadow; @@ -287,7 +287,7 @@ class CaffeOpProp : public OperatorProperty { vector *> bot_blobs, top_blobs; for (int i = 0; i < param_.num_data; ++i) { - TShape tshape = (*in_shape)[i]; + mxnet::TShape tshape = (*in_shape)[i]; if (tshape.ndim() == 0) return false; auto blob_ptr = new Blob(); blob_ptr->Reshape(caffe::TShape2Vector(tshape)); @@ -302,13 +302,13 @@ class CaffeOpProp : public OperatorProperty { // Set weight shape CHECK_EQ(param_.num_weight, caffeOp_->blobs().size()); for (int i = 0; i < param_.num_weight ; ++i) { - TShape tshape = caffe::Vector2TShape(caffeOp_->blobs()[i]->shape()); + mxnet::TShape tshape = caffe::Vector2mxnet::TShape(caffeOp_->blobs()[i]->shape()); SHAPE_ASSIGN_CHECK(*in_shape, i + param_.num_data, tshape); } // Initialize out shapes out_shape->clear(); for (auto blob : top_blobs) { - TShape tshape = caffe::Vector2TShape(blob->shape()); + mxnet::TShape tshape = caffe::Vector2mxnet::TShape(blob->shape()); out_shape->push_back(tshape); } @@ -334,7 +334,7 @@ class CaffeOpProp : public OperatorProperty { return NULL; } - Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + Operator* CreateOperatorEx(Context ctx, mxnet::ShapeVector *in_shape, std::vector *in_type) const override; private: diff --git a/plugin/caffe/caffe_op.cc b/plugin/caffe/caffe_op.cc index 9db9df04068d..715ae0b82d8e 100644 --- a/plugin/caffe/caffe_op.cc +++ b/plugin/caffe/caffe_op.cc @@ -47,10 +47,10 @@ Operator* CreateOp(CaffeOpParam param, int dtype) { } // DO_BIND_DISPATCH comes from static_operator_common.h -Operator *CaffeOpProp::CreateOperatorEx(Context ctx, std::vector *in_shape, +Operator *CaffeOpProp::CreateOperatorEx(Context ctx, mxnet::ShapeVector *in_shape, std::vector *in_type) const { std::vector out_type, aux_type; - std::vector out_shape, aux_shape; + mxnet::ShapeVector out_shape, aux_shape; out_type.resize(this->ListOutputs().size()); out_shape.resize(this->ListOutputs().size()); aux_type.resize(this->ListAuxiliaryStates().size()); diff --git a/plugin/opencv/cv_api.cc b/plugin/opencv/cv_api.cc index 53eb8524c7cf..b0915fd40579 100644 --- a/plugin/opencv/cv_api.cc +++ b/plugin/opencv/cv_api.cc @@ -93,7 +93,7 @@ MXNET_DLL int MXCVImdecode(const unsigned char *img, const mx_uint len, } else { LOG(FATAL) << "Only supports png and jpg."; } - NDArray ndout(TShape(dims, dims+3), Context::CPU(), true, mshadow::kUint8); + NDArray ndout(mxnet::TShape(dims, dims+3), Context::CPU(), true, mshadow::kUint8); unsigned char *img_cpy = new unsigned char[len]; memcpy(img_cpy, img, sizeof(unsigned char)*len); Engine::Get()->PushSync([=](RunContext ctx){ @@ -124,7 +124,7 @@ MXNET_DLL int MXCVResize(NDArrayHandle src, const mx_uint w, const mx_uint h, CHECK_EQ(ndsrc.dtype(), mshadow::kUint8); mx_uint dims[3] = {h, w, ndsrc.shape()[2]}; - NDArray ndout(TShape(dims, dims+3), Context::CPU(), true, mshadow::kUint8); + NDArray ndout(mxnet::TShape(dims, dims+3), Context::CPU(), true, mshadow::kUint8); Engine::Get()->PushSync([=](RunContext ctx){ ndout.CheckAndAlloc(); @@ -156,7 +156,7 @@ MXNET_DLL int MXCVcopyMakeBorder(NDArrayHandle src, int h = ndsrc.shape()[0], w = ndsrc.shape()[1], c = ndsrc.shape()[2]; mx_uint dims[3] = {top+h+bot, left+w+right, c}; - NDArray ndout(TShape(dims, dims+3), Context::CPU(), true, mshadow::kUint8); + NDArray ndout(mxnet::TShape(dims, dims+3), Context::CPU(), true, mshadow::kUint8); Engine::Get()->PushSync([=](RunContext ctx){ ndout.CheckAndAlloc(); diff --git a/plugin/sframe/iter_sframe.cc b/plugin/sframe/iter_sframe.cc index 9f09916b8166..6a6b03f9c2fb 100644 --- a/plugin/sframe/iter_sframe.cc +++ b/plugin/sframe/iter_sframe.cc @@ -50,8 +50,8 @@ struct SFrameParam : public dmlc::Parameter { std::string path_sframe; std::string data_field; std::string label_field; - TShape data_shape; - TShape label_shape; + mxnet::TShape data_shape; + mxnet::TShape label_shape; DMLC_DECLARE_PARAMETER(SFrameParam) { DMLC_DECLARE_FIELD(path_sframe).set_default("") .describe("Dataset Param: path to image dataset sframe"); diff --git a/plugin/torch/torch_criterion-inl.h b/plugin/torch/torch_criterion-inl.h index e0687ab39bff..2138bd8f1335 100644 --- a/plugin/torch/torch_criterion-inl.h +++ b/plugin/torch/torch_criterion-inl.h @@ -42,13 +42,13 @@ namespace mxnet { namespace op { struct TorchCriterionParam : public dmlc::Parameter { std::string lua_string; - TShape label_shape; + mxnet::TShape label_shape; float grad_scale; DMLC_DECLARE_PARAMETER(TorchCriterionParam) { DMLC_DECLARE_FIELD(lua_string) .describe("lua string that is called to generate the torch criterion object"); DMLC_DECLARE_FIELD(label_shape) - .set_default(TShape()) + .set_default(mxnet::TShape()) .enforce_nonzero() .describe("Shape of label (without batch size)."); DMLC_DECLARE_FIELD(grad_scale) @@ -183,18 +183,18 @@ class TorchCriterionProp : public OperatorProperty { return param_.__DICT__(); } - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { + bool InferShape(mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape, + mxnet::ShapeVector *aux_shape) const override { using namespace mshadow; CHECK_EQ(in_shape->size(), 2); - const TShape &dshape = in_shape->at(0); + const mxnet::TShape &dshape = in_shape->at(0); if (dshape.ndim() == 0) return false; std::vector lshape; lshape.push_back(dshape[0]); lshape.insert(lshape.end(), param_.label_shape.data(), param_.label_shape.data() + param_.label_shape.ndim()); - TShape shape(lshape.begin(), lshape.end()); + mxnet::TShape shape(lshape.begin(), lshape.end()); SHAPE_ASSIGN_CHECK(*in_shape, 1, shape); out_shape->clear(); out_shape->push_back(Shape1(dshape[0])); diff --git a/plugin/torch/torch_module-inl.h b/plugin/torch/torch_module-inl.h index 7fb0440aa575..386f0e31fb43 100644 --- a/plugin/torch/torch_module-inl.h +++ b/plugin/torch/torch_module-inl.h @@ -347,9 +347,9 @@ class TorchModuleProp : public OperatorProperty { return param_.__DICT__(); } - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { + bool InferShape(mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape, + mxnet::ShapeVector *aux_shape) const override { if (torchState_ == nullptr) { this->InitTorchState(); } @@ -397,7 +397,7 @@ class TorchModuleProp : public OperatorProperty { THFloatTensor* param = reinterpret_cast(luaT_toudata(L, -1, TorchTensor::TensorType(mshadow::cpu::kDevMask))); long int* size = param->size; // NOLINT(*) - (*in_shape)[index++] = TShape(size, size + THFloatTensor_nDimension(param)); + (*in_shape)[index++] = mxnet::TShape(size, size + THFloatTensor_nDimension(param)); lua_pop(L, 1); } lua_pop(L, 2); @@ -408,7 +408,7 @@ class TorchModuleProp : public OperatorProperty { THFloatTensor* output = reinterpret_cast(luaT_toudata(L, -1, TorchTensor::TensorType(mshadow::cpu::kDevMask))); long int* size = output->size; // NOLINT(*) - (*out_shape)[0] = TShape(size, size + THFloatTensor_nDimension(output)); + (*out_shape)[0] = mxnet::TShape(size, size + THFloatTensor_nDimension(output)); } else { for (uint32_t data_index = 0; data_index < param_.num_outputs; ++data_index) { lua_pushnil(L); @@ -417,7 +417,7 @@ class TorchModuleProp : public OperatorProperty { THFloatTensor* out = reinterpret_cast(luaT_toudata(L, -1, TorchTensor::TensorType(mshadow::cpu::kDevMask))); long int* size = out->size; // NOLINT(*) - (*out_shape)[index++] = TShape(size, size + THFloatTensor_nDimension(out)); + (*out_shape)[index++] = mxnet::TShape(size, size + THFloatTensor_nDimension(out)); } } } diff --git a/plugin/warpctc/warpctc-inl.h b/plugin/warpctc/warpctc-inl.h index 5a540c57940a..37677d21fd14 100644 --- a/plugin/warpctc/warpctc-inl.h +++ b/plugin/warpctc/warpctc-inl.h @@ -247,14 +247,14 @@ class WarpCTCProp : public OperatorProperty { return param_.__DICT__(); } - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { + bool InferShape(mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape, + mxnet::ShapeVector *aux_shape) const override { using namespace mshadow; CHECK_EQ(in_shape->size(), 2) << "Input:[data, label]"; - const TShape &dshape = in_shape->at(0); + const mxnet::TShape &dshape = in_shape->at(0); if (dshape.ndim() == 0) return false; - TShape label_shape(dshape.ndim() - 1); + mxnet::TShape label_shape(dshape.ndim() - 1); label_shape[0] = param_.label_length * (dshape[0] / param_.input_length); SHAPE_ASSIGN_CHECK(*in_shape, warpctc_enum::kLabel, label_shape); @@ -276,7 +276,7 @@ class WarpCTCProp : public OperatorProperty { } std::vector BackwardResource( - const std::vector &in_shape) const override { + const mxnet::ShapeVector &in_shape) const override { return {ResourceRequest::kTempSpace}; } diff --git a/python/README.md b/python/README.md index 1ab7aa4464a3..396d112f687a 100644 --- a/python/README.md +++ b/python/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + MXNet Python Package ==================== This directory and nested files contain MXNet Python package and language binding. diff --git a/python/minpy/README.md b/python/minpy/README.md index 4f028e3b21ad..0278ca1c00fb 100644 --- a/python/minpy/README.md +++ b/python/minpy/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + MXNet Python Package ==================== diff --git a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py index e077824e0226..f9d170d81c13 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py +++ b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py @@ -1537,12 +1537,14 @@ def convert_slice_channel(node, **kwargs): ) return [node] elif squeeze_axis == 0 and num_outputs > 1: + in_shape = kwargs.get('in_shape')[0] + split = in_shape[axis] // num_outputs node = onnx.helper.make_node( "Split", input_nodes, - [name], + [name+'_output'+str(i) for i in range(num_outputs)], axis=axis, - split=[num_outputs], + split=[split for _ in range(num_outputs)], name=name, ) return [node] diff --git a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py index d0d4501d89f4..a7b11fc902db 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py +++ b/python/mxnet/contrib/onnx/mx2onnx/export_onnx.py @@ -262,17 +262,20 @@ def create_onnx_graph_proto(self, sym, params, in_shape, in_type, verbose=False) # If converted node is NodeProto, add it in processed nodes list elif isinstance(converted_node, NodeProto): onnx_processed_nodes.append(converted_node) - node_name = converted_node.name if converted_node.name else converted_node.output[0] - if node_name in graph_outputs: - onnx_processed_outputs.append( - make_tensor_value_info( - name=node_name, - elem_type=in_type, - shape=graph_outputs[node_name] + # some operators have multiple outputs, + # therefore, check all output node names + node_names = list(converted_node.output) + for nodename in node_names: + if nodename in graph_outputs: + onnx_processed_outputs.append( + make_tensor_value_info( + name=nodename, + elem_type=in_type, + shape=graph_outputs[nodename] + ) ) - ) - if verbose: - logging.info("Output node is: %s", converted_node.name) + if verbose: + logging.info("Output node is: %s", nodename) elif isinstance(converted_node, TensorProto): raise ValueError("Did not expect TensorProto") else: diff --git a/python/mxnet/contrib/onnx/onnx2mx/_op_translations.py b/python/mxnet/contrib/onnx/onnx2mx/_op_translations.py index dc00feee815b..24bb727e6401 100644 --- a/python/mxnet/contrib/onnx/onnx2mx/_op_translations.py +++ b/python/mxnet/contrib/onnx/onnx2mx/_op_translations.py @@ -62,11 +62,11 @@ def sample_multinomial(attrs, inputs, proto_obj): new_attrs['dtype'] = TENSOR_TYPE_TO_NP_TYPE[int(attrs.get('dtype', 6))] return 'sample_multinomial', new_attrs, inputs - # Arithmetic Operations def add(attrs, inputs, proto_obj): """Adding two tensors""" new_attr = {} + if 'broadcast' in attrs and attrs['broadcast'] == 1: broadcast_axis = attrs['axis'] op_value = translation_utils._fix_broadcast('broadcast_add', inputs, @@ -77,6 +77,7 @@ def add(attrs, inputs, proto_obj): def subtract(attrs, inputs, proto_obj): """Subtracting two tensors""" new_attr = {} + if 'broadcast' in attrs and attrs['broadcast'] == 1: broadcast_axis = attrs['axis'] op_value = translation_utils._fix_broadcast('broadcast_sub', inputs, @@ -84,10 +85,10 @@ def subtract(attrs, inputs, proto_obj): return op_value, new_attr, inputs return 'broadcast_sub', new_attr, inputs - def multiply(attrs, inputs, proto_obj): """Multiply two tensors""" new_attr = {} + if 'broadcast' in attrs and attrs['broadcast'] == 1: broadcast_axis = attrs['axis'] op_value = translation_utils._fix_broadcast('broadcast_mul', inputs, @@ -98,6 +99,7 @@ def multiply(attrs, inputs, proto_obj): def divide(attrs, inputs, proto_obj): """Divide two tensors""" new_attr = {} + if 'broadcast' in attrs and attrs['broadcast'] == 1: broadcast_axis = attrs['axis'] op_value = translation_utils._fix_broadcast('broadcast_div', inputs, @@ -484,13 +486,15 @@ def split(attrs, inputs, proto_obj): if not split_list: num_outputs = len(proto_obj.model_metadata.get('output_tensor_data')) else: - raise NotImplementedError("Operator {} in MXNet does not support variable splits." - "Tracking the issue to support variable split here: " - "https://github.com/apache/incubator-mxnet/issues/11594" - .format('split')) + if len(set(split_list)) == 1: + num_outputs = len(split_list) + else: + raise NotImplementedError("Operator {} in MXNet does not support variable splits." + "Tracking the issue to support variable split here: " + "https://github.com/apache/incubator-mxnet/issues/11594" + .format('split')) new_attrs['num_outputs'] = num_outputs - return 'split', new_attrs, inputs def _slice(attrs, inputs, proto_obj): diff --git a/python/mxnet/contrib/onnx/onnx2mx/_translation_utils.py b/python/mxnet/contrib/onnx/onnx2mx/_translation_utils.py index 6fd52665ca31..ce55a0b7d66a 100644 --- a/python/mxnet/contrib/onnx/onnx2mx/_translation_utils.py +++ b/python/mxnet/contrib/onnx/onnx2mx/_translation_utils.py @@ -221,7 +221,7 @@ def get_input_shape(sym, proto_obj): model_input_shape = [data[1] for data in proto_obj.model_metadata.get('input_tensor_data')] data_names = [data[0] for data in proto_obj.model_metadata.get('input_tensor_data')] - #creating dummy inputs + # creating dummy inputs inputs = [] for in_shape in model_input_shape: inputs.append(nd.ones(shape=in_shape)) diff --git a/python/mxnet/gluon/contrib/nn/__init__.py b/python/mxnet/gluon/contrib/nn/__init__.py index 62440cda27e2..5eb46f6c08ec 100644 --- a/python/mxnet/gluon/contrib/nn/__init__.py +++ b/python/mxnet/gluon/contrib/nn/__init__.py @@ -17,7 +17,7 @@ # coding: utf-8 # pylint: disable=wildcard-import -"""Contrib recurrent neural network module.""" +"""Contributed neural network modules.""" from . import basic_layers diff --git a/python/mxnet/gluon/contrib/nn/basic_layers.py b/python/mxnet/gluon/contrib/nn/basic_layers.py index 56f0809b345f..ebe136e30208 100644 --- a/python/mxnet/gluon/contrib/nn/basic_layers.py +++ b/python/mxnet/gluon/contrib/nn/basic_layers.py @@ -18,8 +18,10 @@ # coding: utf-8 # pylint: disable= arguments-differ """Custom neural network layers in model_zoo.""" + __all__ = ['Concurrent', 'HybridConcurrent', 'Identity', 'SparseEmbedding', - 'SyncBatchNorm'] + 'SyncBatchNorm', 'PixelShuffle1D', 'PixelShuffle2D', + 'PixelShuffle3D'] import warnings from .... import nd, test_utils @@ -238,3 +240,180 @@ def _get_num_devices(self): def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): return F.contrib.SyncBatchNorm(x, gamma, beta, running_mean, running_var, name='fwd', **self._kwargs) + +class PixelShuffle1D(HybridBlock): + + r"""Pixel-shuffle layer for upsampling in 1 dimension. + + Pixel-shuffling is the operation of taking groups of values along + the *channel* dimension and regrouping them into blocks of pixels + along the ``W`` dimension, thereby effectively multiplying that dimension + by a constant factor in size. + + For example, a feature map of shape :math:`(fC, W)` is reshaped + into :math:`(C, fW)` by forming little value groups of size :math:`f` + and arranging them in a grid of size :math:`W`. + + Parameters + ---------- + factor : int or 1-tuple of int + Upsampling factor, applied to the ``W`` dimension. + + Inputs: + - **data**: Tensor of shape ``(N, f*C, W)``. + Outputs: + - **out**: Tensor of shape ``(N, C, W*f)``. + + Examples + -------- + >>> pxshuf = PixelShuffle1D(2) + >>> x = mx.nd.zeros((1, 8, 3)) + >>> pxshuf(x).shape + (1, 4, 6) + """ + + def __init__(self, factor): + super(PixelShuffle1D, self).__init__() + self._factor = int(factor) + + def hybrid_forward(self, F, x): + """Perform pixel-shuffling on the input.""" + f = self._factor + # (N, C*f, W) + x = F.reshape(x, (0, -4, -1, f, 0)) # (N, C, f, W) + x = F.transpose(x, (0, 1, 3, 2)) # (N, C, W, f) + x = F.reshape(x, (0, 0, -3)) # (N, C, W*f) + return x + + def __repr__(self): + return "{}({})".format(self.__class__.__name__, self._factor) + + +class PixelShuffle2D(HybridBlock): + + r"""Pixel-shuffle layer for upsampling in 2 dimensions. + + Pixel-shuffling is the operation of taking groups of values along + the *channel* dimension and regrouping them into blocks of pixels + along the ``H`` and ``W`` dimensions, thereby effectively multiplying + those dimensions by a constant factor in size. + + For example, a feature map of shape :math:`(f^2 C, H, W)` is reshaped + into :math:`(C, fH, fW)` by forming little :math:`f \times f` blocks + of pixels and arranging them in an :math:`H \times W` grid. + + Pixel-shuffling together with regular convolution is an alternative, + learnable way of upsampling an image by arbitrary factors. It is reported + to help overcome checkerboard artifacts that are common in upsampling with + transposed convolutions (also called deconvolutions). See the paper + `Real-Time Single Image and Video Super-Resolution Using an Efficient + Sub-Pixel Convolutional Neural Network `_ + for further details. + + Parameters + ---------- + factor : int or 2-tuple of int + Upsampling factors, applied to the ``H`` and ``W`` dimensions, + in that order. + + Inputs: + - **data**: Tensor of shape ``(N, f1*f2*C, H, W)``. + Outputs: + - **out**: Tensor of shape ``(N, C, H*f1, W*f2)``. + + Examples + -------- + >>> pxshuf = PixelShuffle2D((2, 3)) + >>> x = mx.nd.zeros((1, 12, 3, 5)) + >>> pxshuf(x).shape + (1, 2, 6, 15) + """ + + def __init__(self, factor): + super(PixelShuffle2D, self).__init__() + try: + self._factors = (int(factor),) * 2 + except TypeError: + self._factors = tuple(int(fac) for fac in factor) + assert len(self._factors) == 2, "wrong length {}".format(len(self._factors)) + + def hybrid_forward(self, F, x): + """Perform pixel-shuffling on the input.""" + f1, f2 = self._factors + # (N, f1*f2*C, H, W) + x = F.reshape(x, (0, -4, -1, f1 * f2, 0, 0)) # (N, C, f1*f2, H, W) + x = F.reshape(x, (0, 0, -4, f1, f2, 0, 0)) # (N, C, f1, f2, H, W) + x = F.transpose(x, (0, 1, 4, 2, 5, 3)) # (N, C, H, f1, W, f2) + x = F.reshape(x, (0, 0, -3, -3)) # (N, C, H*f1, W*f2) + return x + + def __repr__(self): + return "{}({})".format(self.__class__.__name__, self._factors) + + +class PixelShuffle3D(HybridBlock): + + r"""Pixel-shuffle layer for upsampling in 3 dimensions. + + Pixel-shuffling (or voxel-shuffling in 3D) is the operation of taking + groups of values along the *channel* dimension and regrouping them into + blocks of voxels along the ``D``, ``H`` and ``W`` dimensions, thereby + effectively multiplying those dimensions by a constant factor in size. + + For example, a feature map of shape :math:`(f^3 C, D, H, W)` is reshaped + into :math:`(C, fD, fH, fW)` by forming little :math:`f \times f \times f` + blocks of voxels and arranging them in a :math:`D \times H \times W` grid. + + Pixel-shuffling together with regular convolution is an alternative, + learnable way of upsampling an image by arbitrary factors. It is reported + to help overcome checkerboard artifacts that are common in upsampling with + transposed convolutions (also called deconvolutions). See the paper + `Real-Time Single Image and Video Super-Resolution Using an Efficient + Sub-Pixel Convolutional Neural Network `_ + for further details. + + Parameters + ---------- + factor : int or 3-tuple of int + Upsampling factors, applied to the ``D``, ``H`` and ``W`` + dimensions, in that order. + + Inputs: + - **data**: Tensor of shape ``(N, f1*f2*f3*C, D, H, W)``. + Outputs: + - **out**: Tensor of shape ``(N, C, D*f1, H*f2, W*f3)``. + + Examples + -------- + >>> pxshuf = PixelShuffle3D((2, 3, 4)) + >>> x = mx.nd.zeros((1, 48, 3, 5, 7)) + >>> pxshuf(x).shape + (1, 2, 6, 15, 28) + """ + + def __init__(self, factor): + super(PixelShuffle3D, self).__init__() + try: + self._factors = (int(factor),) * 3 + except TypeError: + self._factors = tuple(int(fac) for fac in factor) + assert len(self._factors) == 3, "wrong length {}".format(len(self._factors)) + + def hybrid_forward(self, F, x): + """Perform pixel-shuffling on the input.""" + # `transpose` doesn't support 8D, need other implementation + f1, f2, f3 = self._factors + # (N, C*f1*f2*f3, D, H, W) + x = F.reshape(x, (0, -4, -1, f1 * f2 * f3, 0, 0, 0)) # (N, C, f1*f2*f3, D, H, W) + x = F.swapaxes(x, 2, 3) # (N, C, D, f1*f2*f3, H, W) + x = F.reshape(x, (0, 0, 0, -4, f1, f2*f3, 0, 0)) # (N, C, D, f1, f2*f3, H, W) + x = F.reshape(x, (0, 0, -3, 0, 0, 0)) # (N, C, D*f1, f2*f3, H, W) + x = F.swapaxes(x, 3, 4) # (N, C, D*f1, H, f2*f3, W) + x = F.reshape(x, (0, 0, 0, 0, -4, f2, f3, 0)) # (N, C, D*f1, H, f2, f3, W) + x = F.reshape(x, (0, 0, 0, -3, 0, 0)) # (N, C, D*f1, H*f2, f3, W) + x = F.swapaxes(x, 4, 5) # (N, C, D*f1, H*f2, W, f3) + x = F.reshape(x, (0, 0, 0, 0, -3)) # (N, C, D*f1, H*f2, W*f3) + return x + + def __repr__(self): + return "{}({})".format(self.__class__.__name__, self._factors) diff --git a/python/mxnet/gluon/contrib/rnn/rnn_cell.py b/python/mxnet/gluon/contrib/rnn/rnn_cell.py index 0cbc9eaac375..3bd8e7810978 100644 --- a/python/mxnet/gluon/contrib/rnn/rnn_cell.py +++ b/python/mxnet/gluon/contrib/rnn/rnn_cell.py @@ -22,6 +22,7 @@ from ...rnn import BidirectionalCell, SequentialRNNCell, ModifierCell, HybridRecurrentCell from ...rnn.rnn_cell import _format_sequence, _get_begin_state, _mask_sequence_variable_length from ... import tensor_types +from ....base import _as_list class VariationalDropoutCell(ModifierCell): """ @@ -320,3 +321,117 @@ def hybrid_forward(self, F, inputs, states, i2h_weight, return next_r, [next_r, next_c] # pylint: enable= arguments-differ + + +def dynamic_unroll(cell, inputs, begin_state, drop_inputs=0, drop_outputs=0, + layout='TNC', valid_length=None): + """Unrolls an RNN cell across time steps. + + Currently, 'TNC' is a preferred layout. unroll on the input of this layout + runs much faster. + + Parameters + ---------- + cell : an object whose base class is RNNCell. + The RNN cell to run on the input sequence. + inputs : Symbol + It should have shape (batch_size, length, ...) if `layout` is 'NTC', + or (length, batch_size, ...) if `layout` is 'TNC'. + begin_state : nested list of Symbol + The initial states of the RNN sequence. + drop_inputs : float, default 0. + The dropout rate for inputs. Won't apply dropout if it equals 0. + drop_outputs : float, default 0. + The dropout rate for outputs. Won't apply dropout if it equals 0. + layout : str, optional + `layout` of input symbol. Only used if inputs + is a single Symbol. + valid_length : Symbol, NDArray or None + `valid_length` specifies the length of the sequences in the batch without padding. + This option is especially useful for building sequence-to-sequence models where + the input and output sequences would potentially be padded. + If `valid_length` is None, all sequences are assumed to have the same length. + If `valid_length` is a Symbol or NDArray, it should have shape (batch_size,). + The ith element will be the length of the ith sequence in the batch. + The last valid state will be return and the padded outputs will be masked with 0. + Note that `valid_length` must be smaller or equal to `length`. + + Returns + ------- + outputs : Symbol + the output of the RNN from this unrolling. + + states : list of Symbol + The new state of this RNN after this unrolling. + The type of this symbol is same as the output of `begin_state`. + + Examples + -------- + >>> seq_len = 3 + >>> batch_size = 2 + >>> input_size = 5 + >>> cell = mx.gluon.rnn.LSTMCell(input_size, prefix='rnn_') + >>> cell.initialize(ctx=mx.cpu()) + >>> rnn_data = mx.nd.normal(loc=0, scale=1, shape=(seq_len, batch_size, input_size)) + >>> state_shape = (batch_size, input_size) + >>> states = [mx.nd.normal(loc=0, scale=1, shape=state_shape) for i in range(2)] + >>> valid_length = mx.nd.array([2, 3]) + >>> output, states = mx.gluon.contrib.rnn.rnn_cell.dynamic_unroll(cell, rnn_data, states, + valid_length=valid_length, + layout='TNC') + >>> print(output) + [[[ 0.00767238 0.00023103 0.03973929 -0.00925503 -0.05660512] + [ 0.00881535 0.05428379 -0.02493718 -0.01834097 0.02189514]] + [[-0.00676967 0.01447039 0.01287002 -0.00574152 -0.05734247] + [ 0.01568508 0.02650866 -0.04270559 -0.04328435 0.00904011]] + [[ 0. 0. 0. 0. 0. ] + [ 0.01055336 0.02734251 -0.03153727 -0.03742751 -0.01378113]]] + + """ + + # Merge is always True, so we don't need length. + inputs, axis, F, _ = _format_sequence(0, inputs, layout, True) + if axis != 0: + axes = list(range(len(layout))) + tmp = axes[0] + axes[0] = axes[axis] + axes[axis] = tmp + inputs = F.transpose(inputs, axes=axes) + states = begin_state + + if drop_inputs: + inputs = F.Dropout(inputs, p=drop_inputs, axes=(axis,)) + + if valid_length is None: + def loop_body(inputs, states): + return cell(inputs, states) + else: + zeros = [] + for s in states: + zeros.append(F.zeros_like(s)) + states = list(_as_list(states)) + states.append(F.zeros((1))) + def loop_body(inputs, states): + cell_states = states[:-1] + iter_no = states[-1] + out, new_states = cell(inputs, cell_states) + for i, state in enumerate(cell_states): + new_states[i] = F.where(F.broadcast_greater(valid_length, iter_no), + new_states[i], state) + new_states.append(iter_no + 1) + return out, new_states + + outputs, states = F.contrib.foreach(loop_body, inputs, states) + if drop_outputs: + outputs = F.Dropout(outputs, p=drop_outputs, axes=(axis,)) + if valid_length is not None: + if axis != 0: + outputs = F.transpose(outputs, axes) + outputs = F.SequenceMask(outputs, sequence_length=valid_length, + use_sequence_length=True, axis=axis) + # the last state is the iteration number. We don't need it. + return outputs, states[:-1] + else: + if axis != 0: + outputs = F.transpose(outputs, axes) + return outputs, states diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py index 9d762745a407..934f2d5954c1 100644 --- a/python/mxnet/gluon/data/dataloader.py +++ b/python/mxnet/gluon/data/dataloader.py @@ -169,14 +169,15 @@ def worker_loop_v1(dataset, key_queue, data_queue, batchify_fn): batch = batchify_fn([dataset[i] for i in samples]) data_queue.put((idx, batch)) -def fetcher_loop_v1(data_queue, data_buffer, pin_memory=False, data_buffer_lock=None): +def fetcher_loop_v1(data_queue, data_buffer, pin_memory=False, + pin_device_id=0, data_buffer_lock=None): """Fetcher loop for fetching data from queue and put in reorder dict.""" while True: idx, batch = data_queue.get() if idx is None: break if pin_memory: - batch = _as_in_context(batch, context.cpu_pinned()) + batch = _as_in_context(batch, context.cpu_pinned(pin_device_id)) else: batch = _as_in_context(batch, context.cpu()) if data_buffer_lock is not None: @@ -188,8 +189,8 @@ def fetcher_loop_v1(data_queue, data_buffer, pin_memory=False, data_buffer_lock= class _MultiWorkerIterV1(object): """Internal multi-worker iterator for DataLoader.""" - def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, pin_memory=False, - worker_fn=worker_loop_v1): + def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, + pin_memory=False, pin_device_id=0, worker_fn=worker_loop_v1): assert num_workers > 0, "_MultiWorkerIter is not for {} workers".format(num_workers) self._num_workers = num_workers self._dataset = dataset @@ -218,7 +219,8 @@ def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, pin_memory= self._fetcher = threading.Thread( target=fetcher_loop_v1, - args=(self._data_queue, self._data_buffer, pin_memory, self._data_buffer_lock)) + args=(self._data_queue, self._data_buffer, pin_memory, + pin_device_id, self._data_buffer_lock)) self._fetcher.daemon = True self._fetcher.start() @@ -323,12 +325,15 @@ def default_batchify_fn(data): If ``True``, the dataloader will copy NDArrays into pinned memory before returning them. Copying from CPU pinned memory to GPU is faster than from normal CPU memory. + pin_device_id : int, default 0 + The device id to use for allocating pinned memory if pin_memory is ``True`` """ def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None, last_batch=None, batch_sampler=None, batchify_fn=None, - num_workers=0, pin_memory=False): + num_workers=0, pin_memory=False, pin_device_id=0): self._dataset = dataset self._pin_memory = pin_memory + self._pin_device_id = pin_device_id if batch_sampler is None: if batch_size is None: @@ -365,13 +370,14 @@ def same_process_iter(): for batch in self._batch_sampler: ret = self._batchify_fn([self._dataset[idx] for idx in batch]) if self._pin_memory: - ret = _as_in_context(ret, context.cpu_pinned()) + ret = _as_in_context(ret, context.cpu_pinned(self._pin_device_id)) yield ret return same_process_iter() # multi-worker return _MultiWorkerIterV1(self._num_workers, self._dataset, - self._batchify_fn, self._batch_sampler, self._pin_memory) + self._batchify_fn, self._batch_sampler, + self._pin_memory, self._pin_device_id) def __len__(self): return len(self._batch_sampler) @@ -403,7 +409,7 @@ def _thread_worker_fn(samples, batchify_fn, dataset): class _MultiWorkerIter(object): """Internal multi-worker iterator for DataLoader.""" def __init__(self, worker_pool, batchify_fn, batch_sampler, pin_memory=False, - worker_fn=_worker_fn, prefetch=0, dataset=None): + pin_device_id=0, worker_fn=_worker_fn, prefetch=0, dataset=None): self._worker_pool = worker_pool self._batchify_fn = batchify_fn self._batch_sampler = batch_sampler @@ -413,6 +419,7 @@ def __init__(self, worker_pool, batchify_fn, batch_sampler, pin_memory=False, self._iter = iter(self._batch_sampler) self._worker_fn = worker_fn self._pin_memory = pin_memory + self._pin_device_id = pin_device_id self._dataset = dataset # pre-fetch for _ in range(prefetch): @@ -442,7 +449,7 @@ def __next__(self): ret = self._data_buffer.pop(self._rcvd_idx) batch = pickle.loads(ret.get()) if self._dataset is None else ret.get() if self._pin_memory: - batch = _as_in_context(batch, context.cpu_pinned()) + batch = _as_in_context(batch, context.cpu_pinned(self._pin_device_id)) batch = batch[0] if len(batch) == 1 else batch self._rcvd_idx += 1 return batch @@ -498,6 +505,8 @@ def default_batchify_fn(data): If ``True``, the dataloader will copy NDArrays into pinned memory before returning them. Copying from CPU pinned memory to GPU is faster than from normal CPU memory. + pin_device_id : int, default 0 + The device id to use for allocating pinned memory if pin_memory is ``True`` prefetch : int, default is `num_workers * 2` The number of prefetching batches only works if `num_workers` > 0. If `prefetch` > 0, it allow worker process to prefetch certain batches before @@ -514,9 +523,11 @@ def default_batchify_fn(data): """ def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None, last_batch=None, batch_sampler=None, batchify_fn=None, - num_workers=0, pin_memory=False, prefetch=None, thread_pool=False): + num_workers=0, pin_memory=False, pin_device_id=0, + prefetch=None, thread_pool=False): self._dataset = dataset self._pin_memory = pin_memory + self._pin_device_id = pin_device_id self._thread_pool = thread_pool if batch_sampler is None: @@ -562,13 +573,13 @@ def same_process_iter(): for batch in self._batch_sampler: ret = self._batchify_fn([self._dataset[idx] for idx in batch]) if self._pin_memory: - ret = _as_in_context(ret, context.cpu_pinned()) + ret = _as_in_context(ret, context.cpu_pinned(self._pin_device_id)) yield ret return same_process_iter() # multi-worker return _MultiWorkerIter(self._worker_pool, self._batchify_fn, self._batch_sampler, - pin_memory=self._pin_memory, + pin_memory=self._pin_memory, pin_device_id=self._pin_device_id, worker_fn=_thread_worker_fn if self._thread_pool else _worker_fn, prefetch=self._prefetch, dataset=self._dataset if self._thread_pool else None) diff --git a/python/mxnet/gluon/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py index 5f20d20c02ab..c210081f6071 100644 --- a/python/mxnet/gluon/nn/conv_layers.py +++ b/python/mxnet/gluon/nn/conv_layers.py @@ -673,7 +673,7 @@ def __init__(self, channels, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), class _Pooling(HybridBlock): """Abstract class for different pooling layers.""" def __init__(self, pool_size, strides, padding, ceil_mode, global_pool, - pool_type, count_include_pad=None, **kwargs): + pool_type, layout, count_include_pad=None, **kwargs): super(_Pooling, self).__init__(**kwargs) if strides is None: strides = pool_size @@ -684,6 +684,7 @@ def __init__(self, pool_size, strides, padding, ceil_mode, global_pool, self._kwargs = { 'kernel': pool_size, 'stride': strides, 'pad': padding, 'global_pool': global_pool, 'pool_type': pool_type, + 'layout': layout, 'pooling_convention': 'full' if ceil_mode else 'valid'} if count_include_pad is not None: self._kwargs['count_include_pad'] = count_include_pad @@ -695,7 +696,8 @@ def hybrid_forward(self, F, x): return F.Pooling(x, name='fwd', **self._kwargs) def __repr__(self): - s = '{name}(size={kernel}, stride={stride}, padding={pad}, ceil_mode={ceil_mode})' + s = '{name}(size={kernel}, stride={stride}, padding={pad}, ceil_mode={ceil_mode}' + s += ', global_pool={global_pool}, pool_type={pool_type}, layout={layout})' return s.format(name=self.__class__.__name__, ceil_mode=self._kwargs['pooling_convention'] == 'full', **self._kwargs) @@ -716,7 +718,7 @@ class MaxPool1D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCW' - Dimension ordering of data and weight. Only supports 'NCW' layout for now. + Dimension ordering of data and out ('NCW' or 'NWC'). 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. Pooling is applied on the W dimension. ceil_mode : bool, default False @@ -738,12 +740,13 @@ class MaxPool1D(_Pooling): """ def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', ceil_mode=False, **kwargs): - assert layout == 'NCW', "Only supports 'NCW' layout for now" + assert layout in ('NCW', 'NWC'),\ + "Only NCW and NWC layouts are valid for 1D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,) assert len(pool_size) == 1, "pool_size must be a number or a list of 1 ints" super(MaxPool1D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'max', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'max', layout, **kwargs) class MaxPool2D(_Pooling): @@ -761,7 +764,7 @@ class MaxPool2D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCHW' - Dimension ordering of data and weight. Only supports 'NCHW' layout for now. + Dimension ordering of data and out ('NCHW' or 'NHWC'). 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. ceil_mode : bool, default False @@ -786,12 +789,13 @@ class MaxPool2D(_Pooling): """ def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', ceil_mode=False, **kwargs): - assert layout == 'NCHW', "Only supports 'NCHW' layout for now" + assert layout in ('NCHW', 'NHWC'),\ + "Only NCHW and NHWC layouts are valid for 2D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*2 assert len(pool_size) == 2, "pool_size must be a number or a list of 2 ints" super(MaxPool2D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'max', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'max', layout, **kwargs) class MaxPool3D(_Pooling): @@ -809,7 +813,7 @@ class MaxPool3D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCDHW' - Dimension ordering of data and weight. Only supports 'NCDHW' layout for now. + Dimension ordering of data and out ('NCDHW' or 'NDHWC'). 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. @@ -836,12 +840,13 @@ class MaxPool3D(_Pooling): """ def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, ceil_mode=False, layout='NCDHW', **kwargs): - assert layout == 'NCDHW', "Only supports 'NCDHW' layout for now" + assert layout in ('NCDHW', 'NDHWC'),\ + "Only NCDHW and NDHWC layouts are valid for 3D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*3 assert len(pool_size) == 3, "pool_size must be a number or a list of 3 ints" super(MaxPool3D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'max', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'max', layout, **kwargs) class AvgPool1D(_Pooling): @@ -858,7 +863,7 @@ class AvgPool1D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCW' - Dimension ordering of data and weight. Only supports 'NCW' layout for now. + Dimension ordering of data and out ('NCW' or 'NWC'). 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. padding is applied on 'W' dimension. ceil_mode : bool, default False @@ -882,12 +887,14 @@ class AvgPool1D(_Pooling): """ def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', ceil_mode=False, count_include_pad=True, **kwargs): - assert layout == 'NCW', "Only supports 'NCW' layout for now" + assert layout in ('NCW', 'NWC'),\ + "Only NCW and NWC layouts are valid for 1D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,) assert len(pool_size) == 1, "pool_size must be a number or a list of 1 ints" super(AvgPool1D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'avg', count_include_pad, **kwargs) + pool_size, strides, padding, ceil_mode, False, 'avg', layout, count_include_pad, + **kwargs) class AvgPool2D(_Pooling): @@ -904,7 +911,7 @@ class AvgPool2D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCHW' - Dimension ordering of data and weight. Only supports 'NCHW' layout for now. + Dimension ordering of data and out ('NCHW' or 'NHWC'). 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. ceil_mode : bool, default False @@ -931,12 +938,14 @@ class AvgPool2D(_Pooling): """ def __init__(self, pool_size=(2, 2), strides=None, padding=0, ceil_mode=False, layout='NCHW', count_include_pad=True, **kwargs): - assert layout == 'NCHW', "Only supports 'NCHW' layout for now" + assert layout in ('NCHW', 'NHWC'),\ + "Only NCHW and NHWC layouts are valid for 2D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*2 assert len(pool_size) == 2, "pool_size must be a number or a list of 2 ints" super(AvgPool2D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'avg', count_include_pad, **kwargs) + pool_size, strides, padding, ceil_mode, False, 'avg', layout, count_include_pad, + **kwargs) class AvgPool3D(_Pooling): @@ -953,7 +962,7 @@ class AvgPool3D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. layout : str, default 'NCDHW' - Dimension ordering of data and weight. Can be 'NCDHW', 'NDHWC', etc. + Dimension ordering of data and out ('NCDHW' or 'NDHWC'). 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. @@ -982,12 +991,14 @@ class AvgPool3D(_Pooling): """ def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, ceil_mode=False, layout='NCDHW', count_include_pad=True, **kwargs): - assert layout == 'NCDHW', "Only supports 'NCDHW' layout for now" + assert layout in ('NCDHW', 'NDHWC'),\ + "Only NCDHW and NDHWC layouts are valid for 3D Pooling" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*3 assert len(pool_size) == 3, "pool_size must be a number or a list of 3 ints" super(AvgPool3D, self).__init__( - pool_size, strides, padding, ceil_mode, False, 'avg', count_include_pad, **kwargs) + pool_size, strides, padding, ceil_mode, False, 'avg', layout, count_include_pad, + **kwargs) class GlobalMaxPool1D(_Pooling): @@ -997,7 +1008,7 @@ class GlobalMaxPool1D(_Pooling): Parameters ---------- layout : str, default 'NCW' - Dimension ordering of data and weight. Only supports 'NCW' layout for now. + Dimension ordering of data and out ('NCW' or 'NWC'). 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. Pooling is applied on the W dimension. @@ -1011,9 +1022,10 @@ class GlobalMaxPool1D(_Pooling): when `layout` is `NCW`. """ def __init__(self, layout='NCW', **kwargs): - assert layout == 'NCW', "Only supports 'NCW' layout for now" + assert layout in ('NCW', 'NWC'),\ + "Only NCW and NWC layouts are valid for 1D Pooling" super(GlobalMaxPool1D, self).__init__( - (1,), None, 0, True, True, 'max', **kwargs) + (1,), None, 0, True, True, 'max', layout, **kwargs) class GlobalMaxPool2D(_Pooling): @@ -1023,7 +1035,7 @@ class GlobalMaxPool2D(_Pooling): Parameters ---------- layout : str, default 'NCHW' - Dimension ordering of data and weight. Only supports 'NCHW' layout for now. + Dimension ordering of data and out ('NCHW' or 'NHWC'). 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. @@ -1038,9 +1050,10 @@ class GlobalMaxPool2D(_Pooling): `(batch_size, channels, 1, 1)` when `layout` is `NCHW`. """ def __init__(self, layout='NCHW', **kwargs): - assert layout == 'NCHW', "Only supports 'NCHW' layout for now" + assert layout in ('NCHW', 'NHWC'),\ + "Only NCHW and NHWC layouts are valid for 2D Pooling" super(GlobalMaxPool2D, self).__init__( - (1, 1), None, 0, True, True, 'max', **kwargs) + (1, 1), None, 0, True, True, 'max', layout, **kwargs) class GlobalMaxPool3D(_Pooling): @@ -1050,7 +1063,7 @@ class GlobalMaxPool3D(_Pooling): Parameters ---------- layout : str, default 'NCDHW' - Dimension ordering of data and weight. Only supports 'NCDHW' layout for now. + Dimension ordering of data and out ('NCDHW' or 'NDHWC'). 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. @@ -1066,9 +1079,10 @@ class GlobalMaxPool3D(_Pooling): `(batch_size, channels, 1, 1, 1)` when `layout` is `NCDHW`. """ def __init__(self, layout='NCDHW', **kwargs): - assert layout == 'NCDHW', "Only supports 'NCDHW' layout for now" + assert layout in ('NCDHW', 'NDHWC'),\ + "Only NCDHW and NDHWC layouts are valid for 3D Pooling" super(GlobalMaxPool3D, self).__init__( - (1, 1, 1), None, 0, True, True, 'max', **kwargs) + (1, 1, 1), None, 0, True, True, 'max', layout, **kwargs) class GlobalAvgPool1D(_Pooling): @@ -1077,7 +1091,7 @@ class GlobalAvgPool1D(_Pooling): Parameters ---------- layout : str, default 'NCW' - Dimension ordering of data and weight. Only supports 'NCW' layout for now. + Dimension ordering of data and out ('NCW' or 'NWC'). 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. padding is applied on 'W' dimension. @@ -1090,9 +1104,10 @@ class GlobalAvgPool1D(_Pooling): - **out**: 3D output tensor with shape `(batch_size, channels, 1)`. """ def __init__(self, layout='NCW', **kwargs): - assert layout == 'NCW', "Only supports 'NCW' layout for now" + assert layout in ('NCW', 'NWC'),\ + "Only NCW and NWC layouts are valid for 1D Pooling" super(GlobalAvgPool1D, self).__init__( - (1,), None, 0, True, True, 'avg', **kwargs) + (1,), None, 0, True, True, 'avg', layout, **kwargs) class GlobalAvgPool2D(_Pooling): @@ -1101,7 +1116,7 @@ class GlobalAvgPool2D(_Pooling): Parameters ---------- layout : str, default 'NCHW' - Dimension ordering of data and weight. Only supports 'NCHW' layout for now. + Dimension ordering of data and out ('NCHW' or 'NHWC'). 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. @@ -1116,9 +1131,10 @@ class GlobalAvgPool2D(_Pooling): `(batch_size, channels, 1, 1)` when `layout` is `NCHW`. """ def __init__(self, layout='NCHW', **kwargs): - assert layout == 'NCHW', "Only supports 'NCHW' layout for now" + assert layout in ('NCHW', 'NHWC'),\ + "Only NCHW and NHWC layouts are valid for 2D Pooling" super(GlobalAvgPool2D, self).__init__( - (1, 1), None, 0, True, True, 'avg', **kwargs) + (1, 1), None, 0, True, True, 'avg', layout, **kwargs) class GlobalAvgPool3D(_Pooling): @@ -1127,7 +1143,7 @@ class GlobalAvgPool3D(_Pooling): Parameters ---------- layout : str, default 'NCDHW' - Dimension ordering of data and weight. Can be 'NCDHW', 'NDHWC', etc. + Dimension ordering of data and out ('NCDHW' or 'NDHWC'). 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. @@ -1143,9 +1159,10 @@ class GlobalAvgPool3D(_Pooling): `(batch_size, channels, 1, 1, 1)` when `layout` is `NCDHW`. """ def __init__(self, layout='NCDHW', **kwargs): - assert layout == 'NCDHW', "Only supports 'NCDHW' layout for now" + assert layout in ('NCDHW', 'NDHWC'),\ + "Only NCDHW and NDHWC layouts are valid for 3D Pooling" super(GlobalAvgPool3D, self).__init__( - (1, 1, 1), None, 0, True, True, 'avg', **kwargs) + (1, 1, 1), None, 0, True, True, 'avg', layout, **kwargs) class ReflectionPad2D(HybridBlock): diff --git a/python/mxnet/model.py b/python/mxnet/model.py index c08077cc65f4..efb51096c368 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -181,8 +181,10 @@ def _update_params(param_arrays, grad_arrays, updater, num_device, w, g = p updates[k].append((index*num_device+k, g, w)) for dev_updates in updates: - i, w, g = zip(*dev_updates) - updater(i, w, g) + # update params if param_arrays and grad_arrays are not empty + if dev_updates: + i, w, g = zip(*dev_updates) + updater(i, w, g) def _multiple_callbacks(callbacks, *args, **kwargs): diff --git a/python/mxnet/mxfeatures.py b/python/mxnet/mxfeatures.py deleted file mode 100644 index c546151ab06d..000000000000 --- a/python/mxnet/mxfeatures.py +++ /dev/null @@ -1,103 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# coding: utf-8 -# pylint: disable=not-an-iterable - -"""runtime detection of compile time features in the native library""" - -import ctypes -import enum -from .base import _LIB, check_call, mx_uint - -feature_names = [ - "CUDA", - "CUDNN", - "NCCL", - "CUDA_RTC", - "TENSORRT", - "CPU_SSE", - "CPU_SSE2", - "CPU_SSE3", - "CPU_SSE4_1", - "CPU_SSE4_2", - "CPU_SSE4A", - "CPU_AVX", - "CPU_AVX2", - "OPENMP", - "SSE", - "F16C", - "JEMALLOC", - "BLAS_OPEN", - "BLAS_ATLAS", - "BLAS_MKL", - "BLAS_APPLE", - "LAPACK", - "MKLDNN", - "OPENCV", - "CAFFE", - "PROFILER", - "DIST_KVSTORE", - "CXX14", - "SIGNAL_HANDLER", - "DEBUG" -] - - -Feature = enum.Enum('Feature', {name: index for index, name in enumerate(feature_names)}) - - -def has_feature(feature): - """ - Check the library for compile-time feature at runtime - - Parameters - ---------- - feature : int - An integer representing the feature to check - - Returns - ------- - boolean - True if the feature is enabled, false otherwise - """ - res = ctypes.c_bool() - check_call(_LIB.MXHasFeature(mx_uint(feature), ctypes.byref(res))) - return res.value - - -def features_enabled(): - """ - Returns - ------- - features: list of Feature - list of enabled features in the back-end - """ - res = [] - for f in Feature: - if has_feature(f.value): - res.append(f) - return res - -def features_enabled_str(sep=', '): - """ - Returns - ------- - string with a comma separated list of enabled features in the back-end. For example: - "CPU_SSE, OPENMP, F16C, LAPACK, MKLDNN, OPENCV, SIGNAL_HANDLER, DEBUG" - """ - return sep.join(map(lambda x: x.name, features_enabled())) diff --git a/python/mxnet/ndarray/contrib.py b/python/mxnet/ndarray/contrib.py index 6bbee8a6fbc3..74c355dc1288 100644 --- a/python/mxnet/ndarray/contrib.py +++ b/python/mxnet/ndarray/contrib.py @@ -542,3 +542,29 @@ def isnan(data): """ return data != data + +def adamw_update(weight, grad, mean, var, rescale_grad, lr, eta, beta1=0.9, beta2=0.999, + epsilon=1e-8, wd=0, clip_gradient=-1, out=None, name=None, **kwargs): + if not isinstance(rescale_grad, ndarray.NDArray): + rescale_grad = ndarray.full(shape=(1,), val=rescale_grad, ctx=weight.context) + else: + rescale_grad = rescale_grad.as_in_context(weight.context) + return ndarray._internal._adamw_update(weight=weight, grad=grad, mean=mean, var=var, + rescale_grad=rescale_grad, lr=lr, eta=eta, + beta1=beta1, beta2=beta2, epsilon=epsilon, + wd=wd, clip_gradient=clip_gradient, out=out, + name=name, **kwargs) + +def mp_adamw_update(weight, grad, mean, var, weight32, rescale_grad, lr, eta, beta1=0.9, + beta2=0.999, epsilon=1e-8, wd=0, clip_gradient=-1, out=None, + name=None, **kwargs): + if not isinstance(rescale_grad, ndarray.NDArray): + rescale_grad = ndarray.full(shape=(1,), val=rescale_grad, ctx=weight.context) + else: + rescale_grad = rescale_grad.as_in_context(weight.context) + return ndarray._internal._mp_adamw_update(weight=weight, grad=grad, mean=mean, var=var, + weight32=weight32, + rescale_grad=rescale_grad, lr=lr, eta=eta, + beta1=beta1, beta2=beta2, epsilon=epsilon, + wd=wd, clip_gradient=clip_gradient, out=out, + name=name, **kwargs) diff --git a/python/mxnet/ndarray/register.py b/python/mxnet/ndarray/register.py index 3b19a772411d..05d7f17a8fc1 100644 --- a/python/mxnet/ndarray/register.py +++ b/python/mxnet/ndarray/register.py @@ -167,3 +167,14 @@ def _make_ndarray_function(handle, name, func_name): return ndarray_function _init_op_module('mxnet', 'ndarray', _make_ndarray_function) + +# Update operator documentation with added float support +# Note that we can only do this after the op module is initialized +# Otherwise the backend operators cannot be found +# pylint: disable=wrong-import-position +from .contrib import adamw_update, mp_adamw_update +from ._internal import _adamw_update, _mp_adamw_update +adamw_update.__doc__ = _adamw_update.__doc__.replace("rescale_grad : NDArray", + "rescale_grad : NDArray or float") +mp_adamw_update.__doc__ = _mp_adamw_update.__doc__.replace("rescale_grad : NDArray", + "rescale_grad : NDArray or float") diff --git a/python/mxnet/optimizer/optimizer.py b/python/mxnet/optimizer/optimizer.py index cb52ac54fdab..def2c958ede4 100644 --- a/python/mxnet/optimizer/optimizer.py +++ b/python/mxnet/optimizer/optimizer.py @@ -75,12 +75,11 @@ class Optimizer(object): The initial number of updates. multi_precision : bool, optional, default False - Flag to control the internal precision of the optimizer.:: - - False: results in using the same precision as the weights (default), - True: makes internal 32-bit copy of the weights and applies gradients - in 32-bit precision even if actual weights used in the model have lower precision. - Turning this on can improve convergence and accuracy when training with float16. + Flag to control the internal precision of the optimizer. + False: results in using the same precision as the weights (default), + True: makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. param_dict : dict of int -> gluon.Parameter, default None Dictionary of parameter index to gluon.Parameter, used to lookup parameter attributes @@ -541,12 +540,11 @@ class SGD(Optimizer): Default is True. If True, lazy updates are applied \ if the storage types of weight and grad are both ``row_sparse``. multi_precision: bool, optional - Flag to control the internal precision of the optimizer.:: - - False: results in using the same precision as the weights (default), - True: makes internal 32-bit copy of the weights and applies gradients - in 32-bit precision even if actual weights used in the model have lower precision. - Turning this on can improve convergence and accuracy when training with float16. + Flag to control the internal precision of the optimizer. + False: results in using the same precision as the weights (default), + True: makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. """ def __init__(self, momentum=0.0, lazy_update=True, **kwargs): super(SGD, self).__init__(**kwargs) @@ -790,12 +788,11 @@ class LBSGD(Optimizer): momentum : float, optional The momentum value. multi_precision: bool, optional - Flag to control the internal precision of the optimizer.:: - - False: results in using the same precision as the weights (default), - True: makes internal 32-bit copy of the weights and applies gradients - in 32-bit precision even if actual weights used in the model have lower precision. - Turning this on can improve convergence and accuracy when training with float16. + Flag to control the internal precision of the optimizer. + False: results in using the same precision as the weights (default), + True: makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. warmup_strategy: string ('linear', 'power2', 'sqrt'. , 'lars' default : 'linear') warmup_epochs: unsigned, default: 5 @@ -1031,12 +1028,11 @@ class NAG(Optimizer): momentum : float, optional The momentum value. multi_precision: bool, optional - Flag to control the internal precision of the optimizer.:: - - False: results in using the same precision as the weights (default), - True: makes internal 32-bit copy of the weights and applies gradients - in 32-bit precision even if actual weights used in the model have lower precision. - Turning this on can improve convergence and accuracy when training with float16. + Flag to control the internal precision of the optimizer. + False: results in using the same precision as the weights (default), + True: makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. """ def __init__(self, momentum=0.0, **kwargs): super(NAG, self).__init__(**kwargs) @@ -1095,8 +1091,9 @@ def update(self, index, weight, grad, state): grad = grad * self.rescale_grad if self.clip_gradient is not None: grad = clip(grad, -self.clip_gradient, self.clip_gradient) - weight[:] += - lr/2 * (grad + wd * weight) + normal(0, math.sqrt(lr), shape=weight.shape, - dtype=weight.dtype, ctx=weight.context) + weight[:] += - lr/2 * (grad + wd * weight) + weight[:] += normal(0, math.sqrt(lr), shape=weight.shape, + dtype=weight.dtype, ctx=weight.context) @@ -1376,9 +1373,11 @@ def update(self, index, weight, grad, state): acc_g, acc_delta = state # update g, delta - acc_g[:] = self.rho * acc_g + (1. - self.rho) * grad * grad + acc_g[:] *= self.rho + acc_g[:] += (1. - self.rho) * grad * grad current_delta = sqrt(acc_delta + self.epsilon) / sqrt(acc_g + self.epsilon) * grad - acc_delta[:] = self.rho * acc_delta + (1. - self.rho) * current_delta * current_delta + acc_delta[:] *= self.rho + acc_delta[:] += (1. - self.rho) * current_delta * current_delta # update weight weight[:] -= current_delta + wd * weight @@ -1511,7 +1510,8 @@ def update(self, index, weight, grad, state): # update m_t and u_t m_t, u_t = state - m_t[:] = self.beta1 * m_t + (1. - self.beta1) * grad + m_t[:] *= self.beta1 + m_t[:] += (1. - self.beta1) * grad u_t[:] = maximum(self.beta2 * u_t, NDabs(grad)) # update weight @@ -1574,8 +1574,10 @@ def update(self, index, weight, grad, state): # update m_t and v_t m_t, v_t = state - m_t[:] = self.beta1 * m_t + (1. - self.beta1) * grad - v_t[:] = self.beta2 * v_t + (1. - self.beta2) * grad * grad + m_t[:] *= self.beta1 + m_t[:] += (1. - self.beta1) * grad + v_t[:] *= self.beta2 + v_t[:] += (1. - self.beta2) * grad * grad grad_prime = grad / (1. - self.m_schedule) m_t_prime = m_t / (1. - m_schedule_next) diff --git a/python/mxnet/runtime.py b/python/mxnet/runtime.py new file mode 100644 index 000000000000..e47cca93dace --- /dev/null +++ b/python/mxnet/runtime.py @@ -0,0 +1,100 @@ +# coding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: disable=not-an-iterable + +"""runtime querying of compile time features in the native library""" + +import ctypes +import collections +from .base import _LIB, check_call + +class Feature(ctypes.Structure): + """ + Compile time feature description, member fields: `name` and `enabled`. + """ + _fields_ = [ + ("_name", ctypes.c_char_p), + ("_enabled", ctypes.c_bool) + ] + + @property + def name(self): + """ + Feature name. + """ + return self._name.decode() + + @property + def enabled(self): + """ + True if MXNet was compiled with the given compile-time feature. + """ + return self._enabled + + def __repr__(self): + if self.enabled: + return "✔ {}".format(self.name) + else: + return "✖ {}".format(self.name) + +def feature_list(): + """ + Check the library for compile-time features. The list of features are maintained in libinfo.h and libinfo.cc + + Returns + ------- + list + List of :class:`.Feature` objects + """ + lib_features_c_array = ctypes.POINTER(Feature)() + lib_features_size = ctypes.c_size_t() + check_call(_LIB.MXLibInfoFeatures(ctypes.byref(lib_features_c_array), ctypes.byref(lib_features_size))) + features = [lib_features_c_array[i] for i in range(lib_features_size.value)] + return features + +class Features(collections.OrderedDict): + """ + OrderedDict of name to Feature + """ + def __init__(self): + super(Features, self).__init__([(f.name, f) for f in feature_list()]) + + def __repr__(self): + return str(list(self.values())) + + def is_enabled(self, feature_name): + """ + Check for a particular feature by name + + Parameters + ---------- + feature_name: str + The name of a valid feature as string for example 'CUDA' + + Returns + ------- + Boolean + True if it's enabled, False if it's disabled, RuntimeError if the feature is not known + """ + feature_name = feature_name.upper() + if feature_name not in self: + raise RuntimeError("Feature '{}' is unknown, known features are: {}".format( + feature_name, list(self.keys()))) + return self[feature_name].enabled diff --git a/python/mxnet/symbol/contrib.py b/python/mxnet/symbol/contrib.py index a83227a0261c..d1048df7cd6f 100644 --- a/python/mxnet/symbol/contrib.py +++ b/python/mxnet/symbol/contrib.py @@ -727,3 +727,25 @@ def _union_inputs(*graphs): outputs = [result[i] for i in range(then_num_outputs)] outputs, _ = _regroup(outputs, then_fmt) return outputs + +def adamw_update(weight, grad, mean, var, rescale_grad, lr, eta, beta1=0.9, beta2=0.999, + epsilon=1e-8, wd=0, clip_gradient=-1, out=None, name=None, **kwargs): + if not isinstance(rescale_grad, Symbol): + rescale_grad = symbol.full(shape=(1,), val=rescale_grad) + return symbol._internal._adamw_update(weight=weight, grad=grad, mean=mean, var=var, + rescale_grad=rescale_grad, lr=lr, eta=eta, + beta1=beta1, beta2=beta2, epsilon=epsilon, + wd=wd, clip_gradient=clip_gradient, out=out, + name=name, **kwargs) + +def mp_adamw_update(weight, grad, mean, var, weight32, rescale_grad, lr, eta, beta1=0.9, + beta2=0.999, epsilon=1e-8, wd=0, clip_gradient=-1, out=None, + name=None, **kwargs): + if not isinstance(rescale_grad, Symbol): + rescale_grad = symbol.full(shape=(1,), val=rescale_grad) + return symbol._internal._mp_adamw_update(weight=weight, grad=grad, mean=mean, var=var, + weight32=weight32, + rescale_grad=rescale_grad, lr=lr, eta=eta, + beta1=beta1, beta2=beta2, epsilon=epsilon, + wd=wd, clip_gradient=clip_gradient, out=out, + name=name, **kwargs) diff --git a/python/mxnet/symbol/register.py b/python/mxnet/symbol/register.py index c147914ddb70..15c8e5e1fa68 100644 --- a/python/mxnet/symbol/register.py +++ b/python/mxnet/symbol/register.py @@ -208,3 +208,14 @@ def _make_symbol_function(handle, name, func_name): return symbol_function _init_op_module('mxnet', 'symbol', _make_symbol_function) + +# Update operator documentation with added float support +# Note that we can only do this after the op module is initialized +# Otherwise the backend operators cannot be found +# pylint: disable=wrong-import-position +from .contrib import adamw_update, mp_adamw_update +from ._internal import _adamw_update, _mp_adamw_update +adamw_update.__doc__ = _adamw_update.__doc__.replace("rescale_grad : Symbol", + "rescale_grad : Symbol or float") +mp_adamw_update.__doc__ = _mp_adamw_update.__doc__.replace("rescale_grad : Symbol", + "rescale_grad : Symbol or float") diff --git a/python/mxnet/symbol/symbol.py b/python/mxnet/symbol/symbol.py index 43de0c9d7535..3e3e79ed59f7 100644 --- a/python/mxnet/symbol/symbol.py +++ b/python/mxnet/symbol/symbol.py @@ -882,6 +882,81 @@ def infer_type(self, *args, **kwargs): List of auxiliary state types. The order is same as the order of list_auxiliary_states(). """ + try: + res = self._infer_type_impl(False, *args, **kwargs) + if res[1] is None: + arg_shapes, _, _ = self._infer_type_impl(True, *args, **kwargs) + arg_names = self.list_arguments() + unknowns = [] + for name, dtype in zip(arg_names, arg_shapes): + if not dtype: + if len(unknowns) >= 10: + unknowns.append('...') + break + unknowns.append('%s: %s' % (name, str(dtype))) + warnings.warn( + "Cannot decide type for the following arguments. " + + "Consider providing them as input:\n\t" + + "\n\t".join(unknowns), stacklevel=2) + return res + except MXNetError: + print("infer_type error. Arguments:") + for i, arg in enumerate(args): + print(" #%d: %s" % (i, arg)) + for k, v in kwargs.items(): + print(" %s: %s" % (k, v)) + raise + + def infer_type_partial(self, *args, **kwargs): + """Infers the type partially. + + This functions works the same way as `infer_type`, + except that this function can return partial results. + + In the following example, information about fc2 is not available. So, `infer_shape` + will return a tuple of `None` values but `infer_shape_partial` will return partial values. + + Example + ------- + >>> data = mx.sym.Variable('data') + >>> prev = mx.sym.Variable('prev') + >>> casted_prev = mx.sym.cast(prev, dtype='float32') + >>> out = mx.sym.Activation(data=mx.sym.elemwise_add(data, casted_prev), act_type='relu') + >>> out.list_arguments() + ['data', 'prev'] + >>> out.infer_type(data='float32') + (None, None, None) + >>> out.infer_type_partial(data='float32') + ([numpy.float32, None], [numpy.float32], []) + >>> # infers type if you give information about prev + >>> out.infer_type(data='float32', prev='float16') + ([numpy.float32, numpy.float16], [numpy.float32], []) + + Parameters + ---------- + *args : + Type of known arguments in a positional way. + Unknown type can be marked as None. + + **kwargs : + Keyword arguments of known types. + + Returns + ------- + arg_types : list of numpy.dtype or None + List of argument types. + The order is same as the order of list_arguments(). + out_types : list of numpy.dtype or None + List of output types. + The order is same as the order of list_outputs(). + aux_types : list of numpy.dtype or None + List of auxiliary state types. + The order is same as the order of list_auxiliary_states(). + """ + return self._infer_type_impl(True, *args, **kwargs) + + def _infer_type_impl(self, partial, *args, **kwargs): + """The actual implementation for calling type inference API.""" # pylint: disable=too-many-locals if len(args) != 0 and len(kwargs) != 0: raise ValueError('Can only specify known argument \ @@ -912,7 +987,11 @@ def infer_type(self, *args, **kwargs): aux_type_size = mx_uint() aux_type_data = ctypes.POINTER(ctypes.c_int)() complete = ctypes.c_int() - check_call(_LIB.MXSymbolInferType( + if partial: + infer_func = _LIB.MXSymbolInferTypePartial + else: + infer_func = _LIB.MXSymbolInferType + check_call(infer_func( self.handle, mx_uint(len(sdata)), keys, diff --git a/scala-package/README.md b/scala-package/README.md index be0fc41a5fe4..c7d0cecf15ac 100644 --- a/scala-package/README.md +++ b/scala-package/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + MXNet Package for Scala/Java ===== @@ -162,6 +179,37 @@ mvn deploy -Pstaging Examples & Usage ------- +Assuming you use `mvn install`, you can find the `mxnet-full_scala_version-INTERNAL.jar` e.g. `mxnet-full_2.11-INTERNAL.jar` under the path `incubator-mxnet/scala-package/assembly/target`. + +Adding the following configuration in `pom.xml` +```HTML + + org.apache.mxnet + mxnet-full_2.11-INTERNAL + 1.5.0 + system + path_to_jar/mxnet-full_2.11-INTERNAL.jar + +``` +If you have following error message +``` +Error: A JNI error has occurred, please check your installation and try again +Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/mxnet/NDArray + at java.lang.Class.getDeclaredMethods0(Native Method) + at java.lang.Class.privateGetDeclaredMethods(Class.java:2701) + at java.lang.Class.privateGetMethodRecursive(Class.java:3048) + at java.lang.Class.getMethod0(Class.java:3018) + at java.lang.Class.getMethod(Class.java:1784) + at sun.launcher.LauncherHelper.validateMainClass(LauncherHelper.java:544) + at sun.launcher.LauncherHelper.checkAndLoadMain(LauncherHelper.java:526) +Caused by: java.lang.ClassNotFoundException: org.apache.mxnet.NDArray + at java.net.URLClassLoader.findClass(URLClassLoader.java:381) + at java.lang.ClassLoader.loadClass(ClassLoader.java:424) + at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331) + at java.lang.ClassLoader.loadClass(ClassLoader.java:357) +``` +Please make sure your $CLASSPATH is able to find `mxnet-full_scala_version-INTERNAL.jar`. + - To set up the Scala Project using IntelliJ IDE on macOS follow the instructions [here](https://mxnet.incubator.apache.org/tutorials/scala/mxnet_scala_on_intellij.html). - Several examples on using the Scala APIs are provided in the [Scala Examples Folder](https://github.com/apache/incubator-mxnet/tree/master/scala-package/examples/) diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index 00aa8682f8fa..b93c181bc1d4 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/assembly/src/main/assembly/assembly.xml b/scala-package/assembly/src/main/assembly/assembly.xml index 7525df883dab..4c3d17392624 100644 --- a/scala-package/assembly/src/main/assembly/assembly.xml +++ b/scala-package/assembly/src/main/assembly/assembly.xml @@ -1,3 +1,19 @@ + full @@ -12,6 +28,8 @@ org.scala-lang:* org.scala-lang.modules:* + commons-io:commons-io + commons-codec:commons-codec / true diff --git a/scala-package/assembly/src/main/assembly/javadoc.xml b/scala-package/assembly/src/main/assembly/javadoc.xml index 8f30a261811c..176fa432190c 100644 --- a/scala-package/assembly/src/main/assembly/javadoc.xml +++ b/scala-package/assembly/src/main/assembly/javadoc.xml @@ -1,3 +1,19 @@ + diff --git a/scala-package/assembly/src/main/assembly/source.xml b/scala-package/assembly/src/main/assembly/source.xml index 87fcde360c48..c06786130192 100644 --- a/scala-package/assembly/src/main/assembly/source.xml +++ b/scala-package/assembly/src/main/assembly/source.xml @@ -1,3 +1,19 @@ + diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 4de65c0b361a..5a28a11a0208 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -1,4 +1,20 @@ + @@ -138,11 +154,6 @@ INTERNAL provided - - commons-io - commons-io - 2.1 - org.mockito diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala b/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala index 67809c158aff..50139ec1be22 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/javaapi/NDArray.scala @@ -242,6 +242,8 @@ class NDArray private[mxnet] (val nd: org.apache.mxnet.NDArray ) { this(NDArray.array(arr, shape, ctx)) } + override def toString: String = nd.toString + def serialize(): Array[Byte] = nd.serialize() /** diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml index 4e9da8908be1..74b57077773c 100644 --- a/scala-package/deploy/pom.xml +++ b/scala-package/deploy/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/deploy/src/main/deploy/deploy.xml b/scala-package/deploy/src/main/deploy/deploy.xml index 7bfd20737537..c5be7e2c5ea1 100644 --- a/scala-package/deploy/src/main/deploy/deploy.xml +++ b/scala-package/deploy/src/main/deploy/deploy.xml @@ -1,4 +1,20 @@ + scala-compiler SCALA_VERSION + + commons-codec + commons-codec + 1.10 + + + commons-io + commons-io + 2.1 + diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 30ccfdcce12e..07e430175c6f 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md index 55741024d08b..8a9ed3e1736b 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/objectdetector/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Single Shot Multi Object Detection using Java Inference API In this example, you will learn how to use Java Inference API to run Inference on pre-trained Single Shot Multi Object Detection (SSD) MXNet model. diff --git a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md index 1f2c9e0e813c..09189cb83268 100644 --- a/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md +++ b/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer/predictor/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Image Classification using Java Predictor In this example, you will learn how to use Java Inference API to diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md index 753cb3125410..d45469ac5296 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Benchmarking Scala Inference APIs This folder contains a base class [ScalaInferenceBenchmark](https://github.com/apache/incubator-mxnet/tree/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/benchmark/) and provides a mechanism for benchmarking [MXNet Inference APIs]((https://github.com/apache/incubator-mxnet/tree/master/scala-package/infer)) in Scala. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md index 5e3602e8ab15..3b28e3ef4463 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # CNN Text Classification Example for Scala This is the example using Scala type-safe api doing CNN text classification. This example is only for Illustration and not modeled to achieve the best accuracy. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md index 886fa2cc9d46..a3952aabfb44 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Custom Operator Example for Scala This is the example using Custom Operator for type-safe api of Scala. In the example, a `Softmax` operator is implemented to run the MNIST example. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md index 40db092727c4..a4536a7662e4 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/gan/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # GAN MNIST Example for Scala This is the GAN MNIST Training Example implemented for Scala type-safe api diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md index cec750acdc92..e533130aa71c 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/imclassification/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Image Classification Models This examples contains a number of image classification models that can be run on various datasets. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md index 541e0ce8dd31..e32a66bca832 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Image Classification This folder contains an example for image classification with the [MXNet Scala Infer API](https://github.com/apache/incubator-mxnet/tree/master/scala-package/infer). diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md index 77aec7bb5dee..e5d3bbee0490 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Single Shot Multi Object Detection using Scala Inference API In this example, you will learn how to use Scala Inference API to run Inference on pre-trained Single Shot Multi Object Detection (SSD) MXNet model. diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md index fe849343c9d7..0dc4fb892691 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # Neural Style Example for Scala ## Introduction diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md index 5289fc7b1b4e..2ef2f47c7983 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/rnn/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # RNN Example for MXNet Scala This folder contains the following examples writing in new Scala type-safe API: - [x] LSTM Bucketing diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 565ac6e393a5..ed90d8073675 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index 1721f8cbd403..1254f615c64b 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index c514177db17c..fb9e3d3535fb 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index 52dfde181d72..91a13d0fc521 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -1,4 +1,20 @@ + @@ -20,11 +36,6 @@ INTERNAL provided - - commons-io - commons-io - 2.1 - diff --git a/scala-package/memory-management.md b/scala-package/memory-management.md index 33c36b6e6ab0..b97bcbcb9026 100644 --- a/scala-package/memory-management.md +++ b/scala-package/memory-management.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # JVM Memory Management The Scala and Java bindings of Apache MXNet use native memory (memory from the C++ heap in either RAM or GPU memory) for most of the MXNet objects such as NDArray, Symbol, Executor, KVStore, Data Iterators, etc. The associated Scala classes act only as wrappers. The operations done on these wrapper objects are then directed to the high performance MXNet C++ backend via the Java Native Interface (JNI). Therefore, the bytes are stored in the C++ native heap which allows for fast access. diff --git a/scala-package/mxnet-demo/java-demo/README.md b/scala-package/mxnet-demo/java-demo/README.md index 5dfbf14e8df2..cad52cbc44cb 100644 --- a/scala-package/mxnet-demo/java-demo/README.md +++ b/scala-package/mxnet-demo/java-demo/README.md @@ -1,7 +1,30 @@ + + + + + + + + + + + + + + + + + # MXNet Java Sample Project -This is an project created to use Maven-published Scala/Java package with two Java examples. +This is a project demonstrating how to use the Maven published Scala/Java MXNet package. +The examples provided include: +* NDArray creation +* NDArray operation +* Object Detection using the Inference API +* Image Classification using the Predictor API + ## Setup -You are required to use Maven to build the package with the following commands: +You are required to use Maven to build the package with the following commands under `java-demo`: ``` mvn package ``` @@ -25,16 +48,16 @@ The `SCALA_PKG_PROFILE` should be chosen from `osx-x86_64-cpu`, `linux-x86_64-cp ## Run -### Hello World -The Scala file is being executed using Java. You can execute the helloWorld example as follows: +### NDArrayCreation +The Scala file is being executed using Java. You can execute the `NDArrayCreation` example as follows: ```Bash bash bin/java_sample.sh ``` You can also run the following command manually: ```Bash -java -cp $CLASSPATH sample.HelloWorld +java -cp $CLASSPATH sample.NDArrayCreation ``` -However, you have to define the Classpath before you run the demo code. More information can be found in the `java_sample.sh`. +However, you have to define the Classpath before you run the demo code. More information can be found in `bin/java_sample.sh`. The `CLASSPATH` should point to the jar file you have downloaded. It will load the library automatically and run the example diff --git a/scala-package/mxnet-demo/java-demo/bin/java_sample.sh b/scala-package/mxnet-demo/java-demo/bin/java_sample.sh index 4fb724aca8db..fb1795f20f9d 100755 --- a/scala-package/mxnet-demo/java-demo/bin/java_sample.sh +++ b/scala-package/mxnet-demo/java-demo/bin/java_sample.sh @@ -17,4 +17,4 @@ #!/bin/bash CURR_DIR=$(cd $(dirname $0)/../; pwd) CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* -java -Xmx8G -cp $CLASSPATH mxnet.HelloWorld \ No newline at end of file +java -Xmx8G -cp $CLASSPATH mxnet.NDArrayCreation diff --git a/scala-package/mxnet-demo/java-demo/bin/run_od.sh b/scala-package/mxnet-demo/java-demo/bin/run_od.sh index abd0bf5b1b93..4370518dc8cd 100755 --- a/scala-package/mxnet-demo/java-demo/bin/run_od.sh +++ b/scala-package/mxnet-demo/java-demo/bin/run_od.sh @@ -17,4 +17,4 @@ #!/bin/bash CURR_DIR=$(cd $(dirname $0)/../; pwd) CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* -java -Xmx8G -cp $CLASSPATH mxnet.ObjectDetection \ No newline at end of file +java -Xmx8G -cp $CLASSPATH mxnet.ObjectDetection diff --git a/scala-package/mxnet-demo/java-demo/pom.xml b/scala-package/mxnet-demo/java-demo/pom.xml index 05d04b9e6c14..eb5e043a0dda 100644 --- a/scala-package/mxnet-demo/java-demo/pom.xml +++ b/scala-package/mxnet-demo/java-demo/pom.xml @@ -1,4 +1,20 @@ + @@ -66,6 +82,12 @@ mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} ${mxnet.version} + + org.apache.mxnet + mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} + ${mxnet.version} + sources + commons-io commons-io diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java new file mode 100644 index 000000000000..8cb58da5c2e6 --- /dev/null +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ImageClassification.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mxnet; + +import org.apache.commons.io.FileUtils; +import org.apache.mxnet.infer.javaapi.Predictor; +import org.apache.mxnet.javaapi.*; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +public class ImageClassification { + private static String modelPath; + private static String imagePath; + + private static void downloadUrl(String url, String filePath) { + File tmpFile = new File(filePath); + if (!tmpFile.exists()) { + try { + FileUtils.copyURLToFile(new URL(url), tmpFile); + } catch (Exception exception) { + System.err.println(exception); + } + } + } + + public static void downloadModelImage() { + String tempDirPath = System.getProperty("java.io.tmpdir"); + String baseUrl = "https://s3.us-east-2.amazonaws.com/scala-infer-models"; + downloadUrl(baseUrl + "/resnet-18/resnet-18-symbol.json", + tempDirPath + "/resnet18/resnet-18-symbol.json"); + downloadUrl(baseUrl + "/resnet-18/resnet-18-0000.params", + tempDirPath + "/resnet18/resnet-18-0000.params"); + downloadUrl(baseUrl + "/resnet-18/synset.txt", + tempDirPath + "/resnet18/synset.txt"); + downloadUrl("https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg", + tempDirPath + "/inputImages/resnet18/Pug-Cookie.jpg"); + modelPath = tempDirPath + File.separator + "resnet18/resnet-18"; + imagePath = tempDirPath + File.separator + + "inputImages/resnet18/Pug-Cookie.jpg"; + } + + /** + * Helper class to print the maximum prediction result + * @param probabilities The float array of probability + * @param modelPathPrefix model Path needs to load the synset.txt + */ + private static String printMaximumClass(float[] probabilities, + String modelPathPrefix) throws IOException { + String synsetFilePath = modelPathPrefix.substring(0, + 1 + modelPathPrefix.lastIndexOf(File.separator)) + "/synset.txt"; + BufferedReader reader = new BufferedReader(new FileReader(synsetFilePath)); + ArrayList list = new ArrayList<>(); + String line = reader.readLine(); + + while (line != null){ + list.add(line); + line = reader.readLine(); + } + reader.close(); + + int maxIdx = 0; + for (int i = 1;i probabilities[maxIdx]) { + maxIdx = i; + } + } + + return "Probability : " + probabilities[maxIdx] + " Class : " + list.get(maxIdx) ; + } + + public static void main(String[] args) { + // Download the model and Image + downloadModelImage(); + + // Prepare the model + List context = new ArrayList(); + context.add(Context.cpu()); + List inputDesc = new ArrayList<>(); + Shape inputShape = new Shape(new int[]{1, 3, 224, 224}); + inputDesc.add(new DataDesc("data", inputShape, DType.Float32(), "NCHW")); + Predictor predictor = new Predictor(modelPath, inputDesc, context,0); + + // Prepare data + NDArray nd = Image.imRead(imagePath, 1, true); + nd = Image.imResize(nd, 224, 224, null); + nd = NDArray.transpose(nd, new Shape(new int[]{2, 0, 1}), null)[0]; // HWC to CHW + nd = NDArray.expand_dims(nd, 0, null)[0]; // Add N -> NCHW + nd = nd.asType(DType.Float32()); // Inference with Float32 + + // Predict directly + float[][] result = predictor.predict(new float[][]{nd.toArray()}); + try { + System.out.println("Predict with Float input"); + System.out.println(printMaximumClass(result[0], modelPath)); + } catch (IOException e) { + System.err.println(e); + } + + // predict with NDArray + List ndList = new ArrayList<>(); + ndList.add(nd); + List ndResult = predictor.predictWithNDArray(ndList); + try { + System.out.println("Predict with NDArray"); + System.out.println(printMaximumClass(ndResult.get(0).toArray(), modelPath)); + } catch (IOException e) { + System.err.println(e); + } + } +} diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java new file mode 100644 index 000000000000..32e2d84dcdbf --- /dev/null +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayCreation.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mxnet; + +import org.apache.mxnet.javaapi.*; + +public class NDArrayCreation { + static NDArray$ NDArray = NDArray$.MODULE$; + public static void main(String[] args) { + + // Create new NDArray + NDArray nd = new NDArray(new float[]{2.0f, 3.0f}, new Shape(new int[]{1, 2}), Context.cpu()); + System.out.println(nd); + + // create new Double NDArray + NDArray ndDouble = new NDArray(new double[]{2.0d, 3.0d}, new Shape(new int[]{2, 1}), Context.cpu()); + System.out.println(ndDouble); + + // create ones + NDArray ones = NDArray.ones(Context.cpu(), new int[] {1, 2, 3}); + System.out.println(ones); + + // random + NDArray random = NDArray.random_uniform( + NDArray.new random_uniformParam() + .setLow(0.0f) + .setHigh(2.0f) + .setShape(new Shape(new int[]{10, 10})) + )[0]; + System.out.println(random); + } +} diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/HelloWorld.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayOperation.java similarity index 67% rename from scala-package/mxnet-demo/java-demo/src/main/java/mxnet/HelloWorld.java rename to scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayOperation.java index 71981e2691c5..56a414307f46 100644 --- a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/HelloWorld.java +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/NDArrayOperation.java @@ -14,19 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package mxnet; import org.apache.mxnet.javaapi.*; -import java.util.Arrays; -public class HelloWorld { +public class NDArrayOperation { static NDArray$ NDArray = NDArray$.MODULE$; - public static void main(String[] args) { - System.out.println("Hello World!"); NDArray nd = new NDArray(new float[]{2.0f, 3.0f}, new Shape(new int[]{1, 2}), Context.cpu()); - System.out.println(nd.shape()); - NDArray nd2 = NDArray.dot(NDArray.new dotParam(nd, nd.T()))[0]; - System.out.println(Arrays.toString(nd2.toArray())); + + // Transpose + NDArray ndT = nd.T(); + System.out.println(nd); + System.out.println(ndT); + + // change Data Type + NDArray ndInt = nd.asType(DType.Int32()); + System.out.println(ndInt); + + // element add + NDArray eleAdd = NDArray.elemwise_add(nd, nd, null)[0]; + System.out.println(eleAdd); + + // norm (L2 Norm) + NDArray normed = NDArray.norm(NDArray.new normParam(nd))[0]; + System.out.println(normed); } } diff --git a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java index cfe9b66c4b3f..65fe286aa2c7 100644 --- a/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java +++ b/scala-package/mxnet-demo/java-demo/src/main/java/mxnet/ObjectDetection.java @@ -68,20 +68,18 @@ public static void downloadModelImage() { public static void main(String[] args) { List context = new ArrayList(); - if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && - Integer.valueOf(System.getenv("SCALA_TEST_ON_GPU")) == 1) { - context.add(Context.gpu()); - } else { - context.add(Context.cpu()); - } + context.add(Context.cpu()); downloadModelImage(); + + List> output + = runObjectDetectionSingle(modelPath, imagePath, context); + Shape inputShape = new Shape(new int[] {1, 3, 512, 512}); Shape outputShape = new Shape(new int[] {1, 6132, 6}); int width = inputShape.get(2); int height = inputShape.get(3); - List> output - = runObjectDetectionSingle(modelPath, imagePath, context); String outputStr = "\n"; + for (List ele : output) { for (ObjectDetectorOutput i : ele) { outputStr += "Class: " + i.getClassName() + "\n"; @@ -98,4 +96,4 @@ public static void main(String[] args) { } System.out.println(outputStr); } -} \ No newline at end of file +} diff --git a/scala-package/mxnet-demo/scala-demo/README.md b/scala-package/mxnet-demo/scala-demo/README.md index b994a196b1f6..6c638194687c 100644 --- a/scala-package/mxnet-demo/scala-demo/README.md +++ b/scala-package/mxnet-demo/scala-demo/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Scala Sample Project This is an project created to use Maven-published Scala package with two Scala examples. ## Setup diff --git a/scala-package/mxnet-demo/scala-demo/pom.xml b/scala-package/mxnet-demo/scala-demo/pom.xml index 37d60d133c31..b83ba518aa3c 100644 --- a/scala-package/mxnet-demo/scala-demo/pom.xml +++ b/scala-package/mxnet-demo/scala-demo/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/native/README.md b/scala-package/native/README.md index c87b064fff02..4ca6e5f75eaa 100644 --- a/scala-package/native/README.md +++ b/scala-package/native/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Scala JNI MXNet Scala JNI is a thin wrapper layer of underlying libmxnet.so. diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index 7b776d5b5171..cc8c1cc86daf 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md index e9980f353759..ccbafd46a121 100644 --- a/scala-package/packageTest/README.md +++ b/scala-package/packageTest/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + # MXNet Scala Package Test This is an project created to run the test suite on a fully packaged mxnet jar. The test suite is found locally but mxnet is from the target jarfile. diff --git a/scala-package/packageTest/core/pom.xml b/scala-package/packageTest/core/pom.xml index bdcd7662f082..f21c04a3346c 100644 --- a/scala-package/packageTest/core/pom.xml +++ b/scala-package/packageTest/core/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml index 070b78dda99b..dc3500a00181 100644 --- a/scala-package/packageTest/examples/pom.xml +++ b/scala-package/packageTest/examples/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/packageTest/infer/pom.xml b/scala-package/packageTest/infer/pom.xml index 7c5a096d6e14..137d8aed0d10 100644 --- a/scala-package/packageTest/infer/pom.xml +++ b/scala-package/packageTest/infer/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index f6a16dd77c9d..37b3a9a23931 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 5ba6f1f9f498..62def16627b9 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -1,4 +1,20 @@ + @@ -378,6 +394,11 @@ commons-codec 1.10 + + commons-io + commons-io + 2.1 + org.apache.logging.log4j log4j-core diff --git a/scala-package/spark/README.md b/scala-package/spark/README.md index 503c279038a5..0b18db0a7cb4 100644 --- a/scala-package/spark/README.md +++ b/scala-package/spark/README.md @@ -1,3 +1,20 @@ + + + + + + + + + + + + + + + + + Deep Learning on Spark ===== diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index f2737e9334f4..2acb70b43303 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -1,4 +1,20 @@ + diff --git a/scala-package/spark/src/test/scala/org/apache/mxnet/spark/SharedSparkContext.scala b/scala-package/spark/src/test/scala/org/apache/mxnet/spark/SharedSparkContext.scala index 2efd1814bc90..6d36ca51db90 100644 --- a/scala-package/spark/src/test/scala/org/apache/mxnet/spark/SharedSparkContext.scala +++ b/scala-package/spark/src/test/scala/org/apache/mxnet/spark/SharedSparkContext.scala @@ -80,30 +80,27 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd System.getProperty("user.dir") } - private def getJarFilePath(root: String): String = { - for (platform <- List("linux-x86_64-cpu", "linux-x86_64-gpu", "osx-x86_64-cpu")) { - val jarFiles = new File(s"$root/$platform/target/").listFiles(new FileFilter { - override def accept(pathname: File) = { - pathname.getAbsolutePath.endsWith(".jar") && - !pathname.getAbsolutePath.contains("javadoc") && - !pathname.getAbsolutePath.contains("sources") - } - }) - if (jarFiles != null && jarFiles.nonEmpty) { - return jarFiles.head.getAbsolutePath + private def findJars(root: String): Array[File] = { + val excludedSuffixes = List("bundle", "src", "javadoc", "sources") + new File(root).listFiles(new FileFilter { + override def accept(pathname: File) = { + pathname.getAbsolutePath.endsWith(".jar") && + excludedSuffixes.forall(!pathname.getAbsolutePath.contains(_)) } + }) + } + + private def getJarFilePath(root: String): String = { + val jarFiles = findJars(s"$root/target/") + if (jarFiles != null && jarFiles.nonEmpty) { + jarFiles.head.getAbsolutePath + } else { + null } - null } private def getSparkJar: String = { - val jarFiles = new File(s"$composeWorkingDirPath/target/").listFiles(new FileFilter { - override def accept(pathname: File) = { - pathname.getAbsolutePath.endsWith(".jar") && - !pathname.getAbsolutePath.contains("javadoc") && - !pathname.getAbsolutePath.contains("sources") - } - }) + val jarFiles = findJars(s"$composeWorkingDirPath/target/") if (jarFiles != null && jarFiles.nonEmpty) { jarFiles.head.getAbsolutePath } else { @@ -111,6 +108,9 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd } } + private def getNativeJars(root: String): String = + new File(root).listFiles().map(_.toPath).mkString(",") + protected def buildLeNet(): MXNet = { val workingDir = composeWorkingDirPath val assemblyRoot = s"$workingDir/../assembly" @@ -130,6 +130,8 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd protected def buildMlp(): MXNet = { val workingDir = composeWorkingDirPath val assemblyRoot = s"$workingDir/../assembly" + val nativeRoot = s"$workingDir/../native/target/lib" + new MXNet() .setBatchSize(128) .setLabelName("softmax_label") @@ -139,7 +141,7 @@ trait SharedSparkContext extends FunSuite with BeforeAndAfterEach with BeforeAnd .setNumEpoch(10) .setNumServer(1) .setNumWorker(numWorkers) - .setExecutorJars(s"${getJarFilePath(assemblyRoot)},$getSparkJar") + .setExecutorJars(s"${getJarFilePath(assemblyRoot)},$getSparkJar,${getNativeJars(nativeRoot)}") .setJava("java") .setTimeout(0) } diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index b436e8ca601b..5a7329acaeab 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -43,7 +43,7 @@ #include "mxnet/kvstore.h" #include "mxnet/rtc.h" #include "mxnet/storage.h" -#include "mxnet/mxfeatures.h" +#include "mxnet/libinfo.h" #include "./c_api_common.h" #include "../operator/custom/custom-inl.h" #include "../operator/tensor/matrix_op-inl.h" @@ -87,9 +87,12 @@ inline int MXAPIGetFunctionRegInfo(const FunRegType *e, // NOTE: return value is added in API_END -int MXHasFeature(const mx_uint feature, bool* out) { +int MXLibInfoFeatures(const struct LibFeature **lib_features, size_t *size) { + using namespace features; API_BEGIN(); - *out = features::is_enabled(feature); + LibInfo* lib_info = LibInfo::getInstance(); + *lib_features = lib_info->getFeatures().data(); + *size = lib_info->getFeatures().size(); API_END(); } @@ -167,7 +170,7 @@ int MXNDArrayCreate(const mx_uint *shape, NDArrayHandle *out) { API_BEGIN(); *out = new NDArray( - TShape(shape, shape + ndim), + mxnet::TShape(shape, shape + ndim), Context::Create(static_cast(dev_type), dev_id), delay_alloc != 0); API_END(); @@ -182,7 +185,7 @@ int MXNDArrayCreateEx(const mx_uint *shape, NDArrayHandle *out) { API_BEGIN(); *out = new NDArray( - TShape(shape, shape + ndim), + mxnet::TShape(shape, shape + ndim), Context::Create(static_cast(dev_type), dev_id), delay_alloc != 0, dtype); @@ -203,7 +206,7 @@ int MXNDArrayCreateSparseEx(int storage_type, NDArrayHandle *out) { API_BEGIN(); std::vector aux_types; - std::vector aux_shapes; + mxnet::ShapeVector aux_shapes; auto shape_start = aux_shape; for (size_t i = 0; i < num_aux; i++) { // types @@ -214,7 +217,7 @@ int MXNDArrayCreateSparseEx(int storage_type, } *out = new NDArray( NDArrayStorageType(storage_type), - TShape(shape, shape + ndim), + mxnet::TShape(shape, shape + ndim), Context::Create(static_cast(dev_type), dev_id), delay_alloc != 0, dtype, aux_types, aux_shapes); @@ -430,7 +433,7 @@ MXNET_DLL int MXNDArrayReshape(NDArrayHandle handle, NDArray *ptr = new NDArray(); API_BEGIN(); NDArray *arr = static_cast(handle); - TShape new_shape(dims, dims+ndim); + mxnet::TShape new_shape(dims, dims+ndim); int size = 1; int pos = -1; for (int i = 0; i < ndim; ++i) { @@ -470,7 +473,7 @@ MXNET_DLL int MXNDArrayReshape64(NDArrayHandle handle, nnvm::Tuple shape(dims, dims+ndim); CHECK_GT(arr->shape().Size(), 0) << "Source ndarray's shape is undefined. Input shape: " << arr->shape(); - TShape new_shape = mxnet::op::InferReshapeShape(shape, arr->shape(), reverse); + mxnet::TShape new_shape = mxnet::op::InferReshapeShape(shape, arr->shape(), reverse); *ptr = arr->ReshapeWithRecord(new_shape); *out = ptr; API_END_HANDLE_ERROR(delete ptr); @@ -495,7 +498,7 @@ int MXNDArrayGetShape(NDArrayHandle handle, API_BEGIN(); NDArray *arr = static_cast(handle); if (!arr->is_none()) { - const TShape &s = arr->shape(); + const mxnet::TShape &s = arr->shape(); *out_dim = s.ndim(); std::vector& buffer = ret->arg_shape_buffer; buffer.resize(s.ndim()); @@ -786,7 +789,7 @@ int MXDataIterGetLabel(DataIterHandle handle, NDArrayHandle *out) { NDArray* pndarray = new NDArray(); // temp hack to make label 1D // TODO(tianjun) make label 1D when label_width=0 - TShape shape = db.data[1].shape(); + mxnet::TShape shape = db.data[1].shape(); if (shape[1] == 1) { *pndarray = db.data[1].Reshape(mshadow::Shape1(shape[0])); } else { @@ -1394,6 +1397,6 @@ int MXNDArrayGetSharedMemHandle(NDArrayHandle handle, int* shared_pid, int* shar int MXNDArrayCreateFromSharedMem(int shared_pid, int shared_id, const mx_uint *shape, mx_uint ndim, int dtype, NDArrayHandle *out) { API_BEGIN(); - *out = new NDArray(shared_pid, shared_id, TShape(shape, shape + ndim), dtype); + *out = new NDArray(shared_pid, shared_id, mxnet::TShape(shape, shape + ndim), dtype); API_END(); } diff --git a/src/c_api/c_api_common.h b/src/c_api/c_api_common.h index ecb05bc78ca4..b5adfa37eca9 100644 --- a/src/c_api/c_api_common.h +++ b/src/c_api/c_api_common.h @@ -68,7 +68,7 @@ struct MXAPIThreadLocalEntry { /*! \brief holder for NDArray handles */ std::vector ndinputs, ndoutputs; /*! \brief result holder for returning shapes */ - std::vector arg_shapes, out_shapes, aux_shapes; + mxnet::ShapeVector arg_shapes, out_shapes, aux_shapes; /*! \brief result holder for returning type flags */ std::vector arg_types, out_types, aux_types; /*! \brief result holder for returning storage types */ @@ -83,7 +83,7 @@ struct MXAPIThreadLocalEntry { std::vector save_inputs, save_outputs; // helper function to setup return value of shape array inline static void SetupShapeArrayReturnWithBuffer( - const std::vector &shapes, + const mxnet::ShapeVector &shapes, std::vector *ndim, std::vector *data, std::vector *buffer) { diff --git a/src/c_api/c_api_executor.cc b/src/c_api/c_api_executor.cc index 66566ed703eb..a2e8bb810e6f 100644 --- a/src/c_api/c_api_executor.cc +++ b/src/c_api/c_api_executor.cc @@ -408,10 +408,10 @@ int MXExecutorSimpleBind(SymbolHandle symbol_handle, } // create shape map for in_args and aux_states - std::unordered_map arg_shape_map(num_provided_arg_shapes); + std::unordered_map arg_shape_map(num_provided_arg_shapes); for (mx_uint i = 0; i < num_provided_arg_shapes; ++i) { auto p = arg_shape_map.emplace(provided_arg_shape_names[i], - TShape(provided_arg_shape_data+provided_arg_shape_idx[i], + mxnet::TShape(provided_arg_shape_data+provided_arg_shape_idx[i], provided_arg_shape_data+provided_arg_shape_idx[i+1])); CHECK(p.second) << "Duplicate shapes are provided for argument " << provided_arg_shape_names[i] << " in simple_bind"; @@ -562,10 +562,10 @@ int MXExecutorReshape(int partial_shaping, API_BEGIN(); *out = nullptr; // ensure we can know whether to free executor on early abort // create shape map for in_args and aux_states - std::unordered_map kwargs(num_provided_arg_shapes); + std::unordered_map kwargs(num_provided_arg_shapes); for (mx_uint i = 0; i < num_provided_arg_shapes; ++i) { auto p = kwargs.emplace(provided_arg_shape_names[i], - TShape(provided_arg_shape_data+provided_arg_shape_idx[i], + mxnet::TShape(provided_arg_shape_data+provided_arg_shape_idx[i], provided_arg_shape_data+provided_arg_shape_idx[i+1])); CHECK(p.second) << "Duplicate shapes are provided for argument " << provided_arg_shape_names[i] << " in reshape of executor"; diff --git a/src/c_api/c_api_function.cc b/src/c_api/c_api_function.cc index 7091be2e72c5..50f9b32d6e47 100644 --- a/src/c_api/c_api_function.cc +++ b/src/c_api/c_api_function.cc @@ -37,7 +37,7 @@ namespace custom_function { struct CustomFunctionParam { size_t num_args, num_outs; std::shared_ptr info; - std::vector out_shapes; + std::vector out_shapes; std::vector out_dtypes; }; @@ -64,7 +64,7 @@ std::vector Gradient( OpStatePtr CreateState(const nnvm::NodeAttrs& attrs, Context ctx, - const std::vector& ishape, + const mxnet::ShapeVector& ishape, const std::vector& itype) { LOG(FATAL) << "Not reached"; return OpStatePtr::Create(nullptr); @@ -141,9 +141,9 @@ NNVM_REGISTER_OP(_CustomFunction) const CustomFunctionParam& params = nnvm::get(attrs.parsed); return params.num_outs; }) -.set_attr("FInferShape", - [](const NodeAttrs& attrs, std::vector *in_shape, - std::vector *out_shape) { +.set_attr("FInferShape", + [](const NodeAttrs& attrs, mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape) { const CustomFunctionParam& params = nnvm::get(attrs.parsed); *out_shape = params.out_shapes; return true; diff --git a/src/c_api/c_api_profile.cc b/src/c_api/c_api_profile.cc index dc1b7810147f..0de7b485531c 100644 --- a/src/c_api/c_api_profile.cc +++ b/src/c_api/c_api_profile.cc @@ -52,11 +52,6 @@ struct APICallTimingData { #endif // PROFILE_API_INCLUDE_AS_EVENT }; -template -inline std::unique_ptr make_unique(Args&&... args) { - return std::unique_ptr(new T(std::forward(args)...)); -} - /*! * \brief Per-thread profiling data */ @@ -78,7 +73,7 @@ class ProfilingThreadData { auto iter = tasks_.find(name); if (iter == tasks_.end()) { iter = tasks_.emplace(std::make_pair( - name, make_unique(name, domain))).first; + name, std::make_unique(name, domain))).first; } return iter->second.get(); } @@ -93,7 +88,8 @@ class ProfilingThreadData { // Per-thread so no lock necessary auto iter = events_.find(name); if (iter == events_.end()) { - iter = events_.emplace(std::make_pair(name, make_unique(name))).first; + iter = events_.emplace(std::make_pair(name, + std::make_unique(name))).first; } return iter->second.get(); } diff --git a/src/c_api/c_api_symbolic.cc b/src/c_api/c_api_symbolic.cc index 32b63c11dd9a..e07716267288 100644 --- a/src/c_api/c_api_symbolic.cc +++ b/src/c_api/c_api_symbolic.cc @@ -520,18 +520,18 @@ int MXSymbolInferShape(SymbolHandle sym, MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); API_BEGIN(); nnvm::Graph g = Symbol2Graph(*s); - nnvm::ShapeVector arg_shapes(g.indexed_graph().input_nodes().size(), TShape()); + mxnet::ShapeVector arg_shapes(g.indexed_graph().input_nodes().size(), mxnet::TShape()); if (keys == nullptr && num_args != 0) { std::vector read_only_args = mxnet::ReadOnlyArgIndices(g.indexed_graph()); CHECK_LE(num_args, read_only_args.size()); for (mx_uint i = 0; i < num_args; ++i) { - arg_shapes[read_only_args[i]] = nnvm::ShapeTypeCast( + arg_shapes[read_only_args[i]] = mxnet::ShapeTypeCast( arg_shape_data + arg_ind_ptr[i], arg_shape_data + arg_ind_ptr[i+1]); } } else { - std::unordered_map kwargs; + std::unordered_map kwargs; for (mx_uint i = 0; i < num_args; ++i) { - kwargs[keys[i]] = nnvm::ShapeTypeCast( + kwargs[keys[i]] = mxnet::ShapeTypeCast( arg_shape_data + arg_ind_ptr[i], arg_shape_data + arg_ind_ptr[i+1]); } mxnet::MatchArguments(g.indexed_graph(), kwargs, &arg_shapes, "InferShape"); @@ -544,7 +544,7 @@ int MXSymbolInferShape(SymbolHandle sym, } // copy back - CopyAttr(g.indexed_graph(), g.GetAttr("shape"), + CopyAttr(g.indexed_graph(), g.GetAttr("shape"), &(ret->arg_shapes), &(ret->out_shapes), &(ret->aux_shapes)); // copy data back @@ -638,6 +638,27 @@ int MXSymbolInferType(SymbolHandle sym, API_END(); } +int MXSymbolInferTypePartial(SymbolHandle sym, + mx_uint num_args, + const char** keys, + const int *arg_type_data, + mx_uint *in_type_size, + const int **in_type_data, + mx_uint *out_type_size, + const int **out_type_data, + mx_uint *aux_type_size, + const int **aux_type_data, + int *complete) { + int succ; + *complete = 1; + return MXSymbolInferType(sym, num_args, keys, + arg_type_data, + in_type_size, in_type_data, + out_type_size, out_type_data, + aux_type_size, aux_type_data, + &succ); +} + int MXSymbolGrad(SymbolHandle sym, mx_uint num_wrt, const char** wrt, SymbolHandle* out) { API_BEGIN(); LOG(FATAL) << "not implemented"; diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc index bd599e0b6423..3b9f43d86079 100644 --- a/src/c_api/c_predict_api.cc +++ b/src/c_api/c_predict_api.cc @@ -46,7 +46,7 @@ struct MXAPIPredictor { // auxiliary arrays std::vector aux_arrays; // output shapes - std::vector out_shapes; + mxnet::ShapeVector out_shapes; // uint32_t buffer for output shapes std::vector out_shapes_buffer; // key to arguments @@ -61,7 +61,7 @@ struct MXAPIPredictor { struct MXAPINDList { std::vector keys; - std::vector shapes; + mxnet::ShapeVector shapes; std::vector shapes_buffer; std::vector indptr; std::vector data; @@ -168,17 +168,17 @@ int _CreatePartialOut(const char* symbol_json_str, } // shape inference and bind - std::unordered_map known_shape; + std::unordered_map known_shape; for (mx_uint i = 0; i < num_input_nodes; ++i) { known_shape[std::string(input_keys[i])] = - TShape(input_shape_data + input_shape_indptr[i], + mxnet::TShape(input_shape_data + input_shape_indptr[i], input_shape_data + input_shape_indptr[i + 1]); } std::vector arg_names = sym.ListInputNames(Symbol::kReadOnlyArgs); std::vector aux_names = sym.ListInputNames(Symbol::kAuxiliaryStates); - std::vector out_shapes(sym.ListOutputNames().size()); - std::vector aux_shapes(aux_names.size()); - std::vector arg_shapes; + mxnet::ShapeVector out_shapes(sym.ListOutputNames().size()); + mxnet::ShapeVector aux_shapes(aux_names.size()); + mxnet::ShapeVector arg_shapes; std::unordered_map key2arg; for (size_t i = 0; i < arg_names.size(); ++i) { std::string key = arg_names[i]; @@ -186,7 +186,7 @@ int _CreatePartialOut(const char* symbol_json_str, } try { - std::vector in_shapes; + mxnet::ShapeVector in_shapes; for (std::string key : sym.ListInputNames(Symbol::kAll)) { if (known_shape.count(key) != 0) { in_shapes.push_back(known_shape[key]); @@ -200,7 +200,7 @@ int _CreatePartialOut(const char* symbol_json_str, CHECK(infer_complete) << "The shape information of is not enough to get the shapes"; CopyAttr(g.indexed_graph(), - g.GetAttr("shape"), + g.GetAttr("shape"), &arg_shapes, &out_shapes, &aux_shapes); } catch (const mxnet::op::InferShapeError &err) { throw dmlc::Error(err.msg); @@ -348,22 +348,22 @@ int MXPredReshape(mx_uint num_input_nodes, API_BEGIN(); // shape inference - std::unordered_map new_shape; + std::unordered_map new_shape; for (mx_uint i = 0; i < num_input_nodes; ++i) { new_shape[std::string(input_keys[i])] = - TShape(input_shape_data + input_shape_indptr[i], + mxnet::TShape(input_shape_data + input_shape_indptr[i], input_shape_data + input_shape_indptr[i + 1]); } ret->sym = p->sym; std::vector arg_names = ret->sym.ListInputNames(Symbol::kReadOnlyArgs); std::vector aux_names = ret->sym.ListInputNames(Symbol::kAuxiliaryStates); - std::vector out_shapes(ret->sym.ListOutputNames().size()); - std::vector aux_shapes(aux_names.size()); - std::vector arg_shapes; + mxnet::ShapeVector out_shapes(ret->sym.ListOutputNames().size()); + mxnet::ShapeVector aux_shapes(aux_names.size()); + mxnet::ShapeVector arg_shapes; ret->key2arg = p->key2arg; try { - std::vector in_shapes; + mxnet::ShapeVector in_shapes; in_shapes.reserve(arg_names.size()); for (std::string key : ret->sym.ListInputNames(Symbol::kAll)) { if (new_shape.count(key) != 0) { @@ -378,7 +378,7 @@ int MXPredReshape(mx_uint num_input_nodes, CHECK(infer_complete) << "The shape information of is not enough to get the shapes"; CopyAttr(g.indexed_graph(), - g.GetAttr("shape"), + g.GetAttr("shape"), &arg_shapes, &out_shapes, &aux_shapes); } catch (const mxnet::op::InferShapeError &err) { throw dmlc::Error(err.msg); @@ -387,7 +387,7 @@ int MXPredReshape(mx_uint num_input_nodes, ret->arg_arrays = p->arg_arrays; ret->ctx = p->ctx; for (size_t i=0; i < arg_names.size(); ++i) { - TShape newShape = arg_shapes[i]; + mxnet::TShape newShape = arg_shapes[i]; NDArray &arr = p->arg_arrays[i]; if (new_shape.count(arg_names[i]) != 0) { ret->arg_arrays[i].ReshapeAndAlloc(newShape); @@ -399,7 +399,7 @@ int MXPredReshape(mx_uint num_input_nodes, } for (size_t i=0; i < aux_names.size(); ++i) { - TShape newShape = aux_shapes[i]; + mxnet::TShape newShape = aux_shapes[i]; NDArray &arr = p->aux_arrays[i]; CHECK_EQ(newShape.Size(), arr.shape().Size()) << "aux " << aux_names[i] @@ -435,7 +435,7 @@ int MXPredGetOutputShape(PredictorHandle handle, CHECK_LT(out_index, p->out_arrays.size()) << "Index exceed number of outputs"; - const TShape& s = p->out_shapes[out_index]; + const mxnet::TShape& s = p->out_shapes[out_index]; p->out_shapes_buffer.resize(s.ndim()); nnvm::ShapeTypeCast(s.begin(), s.end(), p->out_shapes_buffer.data()); *shape_data = p->out_shapes_buffer.data(); @@ -509,7 +509,7 @@ int MXNDListCreate(const char* nd_file_bytes, } ret->indptr.push_back(0); for (auto &array : arrays) { - TShape shape = array.shape(); + mxnet::TShape shape = array.shape(); size_t begin = ret->data.size(); size_t size = shape.Size(); ret->shapes.push_back(shape); @@ -534,7 +534,7 @@ int MXNDListGet(NDListHandle handle, << "Index out of range"; *out_key = p->keys[index].c_str(); *out_data = dmlc::BeginPtr(p->data) + p->indptr[index]; - const TShape& s = p->shapes[index]; + const mxnet::TShape& s = p->shapes[index]; p->shapes_buffer.resize(s.ndim()); nnvm::ShapeTypeCast(s.begin(), s.end(), p->shapes_buffer.data()); *out_shape = p->shapes_buffer.data(); diff --git a/src/common/exec_utils.h b/src/common/exec_utils.h index 8de6f65253aa..279ecbd67f09 100644 --- a/src/common/exec_utils.h +++ b/src/common/exec_utils.h @@ -284,7 +284,7 @@ inline std::string storage_str(int storage_id) { */ inline void LogMemoryPlan(const nnvm::Graph& g) { const auto &idx = g.indexed_graph(); - const auto& vshape = g.GetAttr("shape"); + const auto& vshape = g.GetAttr("shape"); const auto& vtype = g.GetAttr("dtype"); const auto& vstorage = g.GetAttr("storage_id"); // find node range @@ -373,13 +373,13 @@ inline void LogInferStorage(const nnvm::Graph& g) { // prints a helpful message after shape inference errors in executor. inline void HandleInferShapeError(const size_t num_forward_inputs, const nnvm::IndexedGraph& idx, - const nnvm::ShapeVector& inferred_shapes) { + const mxnet::ShapeVector& inferred_shapes) { int cnt = 10; std::ostringstream oss; for (size_t i = 0; i < num_forward_inputs; ++i) { const uint32_t nid = idx.input_nodes().at(i); const uint32_t eid = idx.entry_id(nid, 0); - const TShape& inferred_shape = inferred_shapes[eid]; + const mxnet::TShape& inferred_shape = inferred_shapes[eid]; if (inferred_shape.ndim() == 0 || inferred_shape.Size() == 0U) { const std::string& arg_name = idx[nid].source->attrs.name; oss << arg_name << ": " << inferred_shape << ", "; @@ -451,7 +451,7 @@ inline void HandleInferStorageTypeError(const size_t num_forward_inputs, * if enable_row_sparse_sharing is `True`, otherwise default storage only. */ inline NDArray ReshapeOrCreate(const std::string& name, - const TShape& dest_arg_shape, + const mxnet::TShape& dest_arg_shape, const int dest_arg_dtype, const NDArrayStorageType dest_arg_stype, const Context& ctx, diff --git a/src/common/object_pool.h b/src/common/object_pool.h index 576ff9aea1cc..362553313b85 100644 --- a/src/common/object_pool.h +++ b/src/common/object_pool.h @@ -132,10 +132,9 @@ struct ObjectPoolAllocatable { template ObjectPool::~ObjectPool() { - // TODO(hotpxl): mind destruction order - // for (auto i : allocated_) { - // free(i); - // } + for (auto i : allocated_) { + free(i); + } } template diff --git a/src/common/serialization.h b/src/common/serialization.h index 8a1bcc6e6ed2..8192ee210a1c 100644 --- a/src/common/serialization.h +++ b/src/common/serialization.h @@ -30,7 +30,6 @@ #include #include #include -#include #include #include diff --git a/src/common/utils.h b/src/common/utils.h index b902b38b90ce..8e6966952890 100644 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -118,10 +118,10 @@ void CheckFormatCSRImpl(const RunContext &rctx, const NDArray &input, using namespace op::mxnet_op; CHECK_EQ(input.storage_type(), kCSRStorage) << "CheckFormatCSRImpl is for CSRNDArray"; - const TShape shape = input.shape(); - const TShape idx_shape = input.aux_shape(csr::kIdx); - const TShape indptr_shape = input.aux_shape(csr::kIndPtr); - const TShape storage_shape = input.storage_shape(); + const mxnet::TShape shape = input.shape(); + const mxnet::TShape idx_shape = input.aux_shape(csr::kIdx); + const mxnet::TShape indptr_shape = input.aux_shape(csr::kIndPtr); + const mxnet::TShape storage_shape = input.storage_shape(); if ((shape.ndim() != 2) || (idx_shape.ndim() != 1 || indptr_shape.ndim() != 1 || storage_shape.ndim() != 1) || (indptr_shape[0] != shape[0] + 1) || @@ -172,7 +172,7 @@ void CheckFormatRSPImpl(const RunContext &rctx, const NDArray &input, using namespace op::mxnet_op; CHECK_EQ(input.storage_type(), kRowSparseStorage) << "CheckFormatRSPImpl is for RSPNDArray"; - const TShape idx_shape = input.aux_shape(rowsparse::kIdx); + const mxnet::TShape idx_shape = input.aux_shape(rowsparse::kIdx); if (idx_shape[0] != input.storage_shape()[0]) { MSHADOW_TYPE_SWITCH(err_cpu.type_flag_, DType, { DType* err = err_cpu.dptr(); @@ -689,7 +689,7 @@ MSHADOW_XINLINE int ilog2ui(unsigned int a) { /*! * \brief Return an NDArray of all zeros. */ -inline NDArray InitZeros(const NDArrayStorageType stype, const TShape &shape, +inline NDArray InitZeros(const NDArrayStorageType stype, const mxnet::TShape &shape, const Context &ctx, const int dtype) { // NDArray with default storage if (stype == kDefaultStorage) { @@ -704,7 +704,7 @@ inline NDArray InitZeros(const NDArrayStorageType stype, const TShape &shape, /*! * \brief Helper to add a NDArray of zeros to a std::vector. */ -inline void EmplaceBackZeros(const NDArrayStorageType stype, const TShape &shape, +inline void EmplaceBackZeros(const NDArrayStorageType stype, const mxnet::TShape &shape, const Context &ctx, const int dtype, std::vector *vec) { // NDArray with default storage diff --git a/src/engine/naive_engine.cc b/src/engine/naive_engine.cc index 05b72d2a6fde..db4491981bdd 100644 --- a/src/engine/naive_engine.cc +++ b/src/engine/naive_engine.cc @@ -62,6 +62,8 @@ class NaiveEngine final : public Engine { }; NaiveEngine() { + objpool_opr_ref_ = common::ObjectPool::_GetSharedRef(); + objpool_var_ref_ = common::ObjectPool::_GetSharedRef(); } // virtual destructor virtual ~NaiveEngine() { @@ -74,6 +76,12 @@ class NaiveEngine final : public Engine { streams_[i] = nullptr; } } + for (size_t i = 0; i < aux_streams_.size(); ++i) { + if (aux_streams_[i] != nullptr) { + delete aux_streams_[i]; + aux_streams_[i] = nullptr; + } + } #endif } @@ -169,16 +177,18 @@ class NaiveEngine final : public Engine { MSHADOW_CATCH_ERROR(mshadow::SetDevice(exec_ctx.dev_id)); if (streams_.size() <= dev_id) { streams_.resize(dev_id + 1, nullptr); + aux_streams_.resize(dev_id + 1, nullptr); } if (streams_[dev_id] == nullptr) { streams_[dev_id] = mshadow::NewStream(true, MXNET_USE_CUDNN != 0, dev_id); + aux_streams_[dev_id] = new GPUAuxStream(streams_[dev_id]); } - exec_fun(RunContext{exec_ctx, streams_[dev_id]}, callback); + exec_fun(RunContext{exec_ctx, streams_[dev_id], aux_streams_[dev_id], false}, callback); #else LOG(FATAL) << "GPU is not enabled"; #endif } else { - exec_fun(RunContext{exec_ctx, &cpu_stream_}, callback); + exec_fun(RunContext{exec_ctx, &cpu_stream_, nullptr, false}, callback); } CHECK(this->req_completed_) << "NaiveEngine only support synchronize Push so far"; @@ -220,6 +230,17 @@ class NaiveEngine final : public Engine { mshadow::Stream cpu_stream_; // GPU streams std::vector*> streams_; +#if MXNET_USE_CUDA + // GPU auxiliary streams + std::vector aux_streams_; +#endif +/*! + * \brief Holding a shared_ptr to the object pool to prevent it from being destructed too early + * See also #309 (https://github.com/dmlc/mxnet/issues/309) and similar fix in threaded_engine.h. + * Without this, segfaults seen on CentOS7 in test_operator_gpu.py:test_convolution_multiple_streams + */ + std::shared_ptr > objpool_opr_ref_; + std::shared_ptr > objpool_var_ref_; }; // class NaiveEngine Engine *CreateNaiveEngine() { diff --git a/src/engine/stream_manager.h b/src/engine/stream_manager.h index 8d44d9c15531..42d03e55a275 100644 --- a/src/engine/stream_manager.h +++ b/src/engine/stream_manager.h @@ -55,6 +55,8 @@ class StreamManager { #if MXNET_USE_CUDA std::array*, kStreams>, kNumGpus> gpu_streams_; + std::array, kNumGpus> + gpu_aux_streams_; std::array*, kNumGpus> gpu_io_streams_; std::array gpu_cnt_; #endif // MXNET_USE_CUDA @@ -67,7 +69,7 @@ RunContext StreamManager::GetRunContext( RunContext ret; switch (ctx.dev_mask()) { case cpu::kDevMask: - ret = RunContext{ctx, nullptr, false}; + ret = RunContext{ctx, nullptr, nullptr, false}; break; case gpu::kDevMask: { #if MXNET_USE_CUDA @@ -77,8 +79,13 @@ RunContext StreamManager::GetRunContext( auto&& counter = gpu_cnt_.at(ctx.dev_id); if (counter == -1) { mxnet::common::cuda::DeviceStore device_store(ctx.dev_id); - for (auto&& i : gpu_streams_.at(ctx.dev_id)) { - i = mshadow::NewStream(true, MXNET_USE_CUDNN != 0, ctx.dev_id); + for (auto&& primary_stream : gpu_streams_.at(ctx.dev_id)) { + primary_stream = mshadow::NewStream(true, MXNET_USE_CUDNN != 0, ctx.dev_id); + } + int idx = 0; + for (auto&& aux_stream : gpu_aux_streams_.at(ctx.dev_id)) { + auto primary_stream = gpu_streams_.at(ctx.dev_id).at(idx++); + aux_stream = new GPUAuxStream(primary_stream); } counter = 0; } @@ -87,6 +94,7 @@ RunContext StreamManager::GetRunContext( } ret = RunContext{ctx, gpu_streams_.at(ctx.dev_id).at(use_counter), + gpu_aux_streams_.at(ctx.dev_id).at(use_counter), false}; break; #else @@ -105,7 +113,7 @@ RunContext StreamManager::GetIORunContext( RunContext ret; switch (ctx.dev_mask()) { case cpu::kDevMask: - ret = RunContext{ctx, nullptr, false}; + ret = RunContext{ctx, nullptr, nullptr, false}; break; case gpu::kDevMask: { #if MXNET_USE_CUDA @@ -116,7 +124,7 @@ RunContext StreamManager::GetIORunContext( gpu_io_streams_.at(ctx.dev_id) = mshadow::NewStream(false, false, ctx.dev_id); } } - ret = RunContext{ctx, gpu_io_streams_.at(ctx.dev_id), false}; + ret = RunContext{ctx, gpu_io_streams_.at(ctx.dev_id), nullptr, false}; break; #else LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR; @@ -145,9 +153,12 @@ void StreamManager::Finalize() { #if MXNET_USE_CUDA for (std::size_t i = 0; i < kNumGpus; ++i) { if (gpu_cnt_.at(i) != -1) { - for (auto&& j : gpu_streams_.at(i)) { + for (auto&& primary_stream : gpu_streams_.at(i)) { // Catch exception for CUDA driver shutdown - MSHADOW_CATCH_ERROR(mshadow::DeleteStream(j)); + MSHADOW_CATCH_ERROR(mshadow::DeleteStream(primary_stream)); + } + for (auto&& aux_stream : gpu_aux_streams_.at(i)) { + delete aux_stream; } gpu_cnt_.at(i) = -1; } diff --git a/src/engine/threaded_engine.cc b/src/engine/threaded_engine.cc index 6a6004011db1..b5897a1ca9cd 100644 --- a/src/engine/threaded_engine.cc +++ b/src/engine/threaded_engine.cc @@ -281,7 +281,7 @@ void ThreadedEngine::DeleteOperator(OprHandle op) { this->PushAsync([threaded_opr](RunContext, CallbackOnComplete on_complete) { ThreadedOpr::Delete(threaded_opr); on_complete(); - }, Context::CPU(), {}, deps, FnProperty::kAsync, 0, + }, Context::CPU(), {}, deps, FnProperty::kDeleteVar, 0, "DeleteOperator"); } diff --git a/src/engine/threaded_engine.h b/src/engine/threaded_engine.h index 4a2d4196dcd3..ab06ca1b9b47 100644 --- a/src/engine/threaded_engine.h +++ b/src/engine/threaded_engine.h @@ -352,7 +352,8 @@ class ThreadedEngine : public Engine { LOG(INFO) << "ExecuteOprBlock " << opr_block << "shutdown_phase=" << shutdown_phase_; } - if (!shutdown_phase_) { + // still run cleanup in shutdown_phase + if (!shutdown_phase_ || threaded_opr->prop == FnProperty::kDeleteVar) { try { OnStart(threaded_opr); if (debug_info) { diff --git a/src/engine/threaded_engine_perdevice.cc b/src/engine/threaded_engine_perdevice.cc index b6537dabb638..bcb101e9e1bb 100644 --- a/src/engine/threaded_engine_perdevice.cc +++ b/src/engine/threaded_engine_perdevice.cc @@ -99,7 +99,7 @@ class ThreadedEnginePerDevice : public ThreadedEngine { MSHADOW_CATCH_ERROR(mshadow::SetDevice(ctx.dev_id)); #endif } - this->ExecuteOprBlock(RunContext{ctx, nullptr}, opr_block); + this->ExecuteOprBlock(RunContext{ctx, nullptr, nullptr, false}, opr_block); } else { if (ctx.dev_mask() == Context::kCPU) { // CPU execution. @@ -244,7 +244,8 @@ class ThreadedEnginePerDevice : public ThreadedEngine { this->is_worker_ = true; #if MXNET_USE_CUDA CHECK(block != nullptr); - mshadow::Stream *stream; + mshadow::Stream *stream = nullptr; + GPUAuxStream *aux_stream = nullptr; do { ThreadPool::SetReadyOnDestroy setReady(ready_event); // allocate stream @@ -253,11 +254,12 @@ class ThreadedEnginePerDevice : public ThreadedEngine { stream = mshadow::NewStream(false, false, ctx.dev_id); } else { stream = mshadow::NewStream(true, MXNET_USE_CUDNN != 0, ctx.dev_id); + aux_stream = new GPUAuxStream(stream); } } while (false); // execute task OprBlock* opr_block; - RunContext run_ctx{ctx, stream}; + RunContext run_ctx{ctx, stream, aux_stream, false}; auto* task_queue = &(block->task_queue); // Don't eat up omp threads for GPU jobs. They're probably best used elsewhere, @@ -269,6 +271,8 @@ class ThreadedEnginePerDevice : public ThreadedEngine { } // Catch exception for CUDA driver shutdown MSHADOW_CATCH_ERROR(mshadow::DeleteStream(stream)); + if (aux_stream != nullptr) + delete aux_stream; #else ready_event->signal(); #endif @@ -283,7 +287,7 @@ class ThreadedEnginePerDevice : public ThreadedEngine { const std::shared_ptr& ready_event) { this->is_worker_ = true; auto* task_queue = &(block->task_queue); - RunContext run_ctx{ctx, nullptr}; + RunContext run_ctx{ctx, nullptr, nullptr, false}; // execute task OprBlock* opr_block; diff --git a/src/executor/attach_op_execs_pass.cc b/src/executor/attach_op_execs_pass.cc index fe3a6bd3d09f..b04d132ee9f6 100644 --- a/src/executor/attach_op_execs_pass.cc +++ b/src/executor/attach_op_execs_pass.cc @@ -263,7 +263,7 @@ class FComputeExExecutor : public OpExecutor { void CreateOpExecs(const Graph& g, OpExecVector* p_ret, size_t i) { using nnvm::DTypeVector; - using nnvm::ShapeVector; + using mxnet::ShapeVector; using nnvm::FMutateInputs; static auto& fcreate_op_state = nnvm::Op::GetAttr("FCreateOpState"); @@ -272,7 +272,7 @@ void CreateOpExecs(const Graph& g, OpExecVector* p_ret, size_t i) { static auto& is_layer_backward = nnvm::Op::GetAttr("TIsLayerOpBackward"); const auto& vdtype = g.GetAttr("dtype"); - const auto& vshape = g.GetAttr("shape"); + const auto& vshape = g.GetAttr("shape"); const auto& vctx = g.GetAttr("context"); const auto& dispatch_modes = g.GetAttr("dispatch_mode"); // get the graph @@ -293,7 +293,7 @@ void CreateOpExecs(const Graph& g, OpExecVector* p_ret, size_t i) { } CHECK(dispatch_modes[i] != DispatchMode::kUndefined); if (fcreate_op_state.count(op)) { - std::vector ishape; + mxnet::ShapeVector ishape; std::vector itype; for (const auto& e : inode.inputs) { ishape.emplace_back(vshape[idx.entry_id(e)]); diff --git a/src/executor/exec_pass.h b/src/executor/exec_pass.h index 52f7c790c77e..dd4132301346 100644 --- a/src/executor/exec_pass.h +++ b/src/executor/exec_pass.h @@ -173,7 +173,7 @@ Graph DetectInplaceAddTo(Graph g); * The index of ShapeVector is given by graph.indexed_graph().entry_id. */ Graph InferShape(Graph&& graph, - nnvm::ShapeVector&& shape_inputs = nnvm::ShapeVector(), + mxnet::ShapeVector&& shape_inputs = mxnet::ShapeVector(), const std::string& shape_attr_key = ""); /*! @@ -217,4 +217,55 @@ std::vector> GetTrtCompatibleSubsets(const Graph } // namespace exec } // namespace mxnet +namespace nnvm { +namespace pass { +/*! + * \brief Get the gradient graph whose outputs are gradients of xs wrt to ys. + * \param graph The input graph. + * \param ys The entries we want to take gradient from. + * \param xs The input to take gradient with respect to. + * \param ys_out_grad The symbol for additional gradient to be propagate back to y. + * \param aggregate_fun Aggregation function applied to aggregate the inputs. + * \param mirror_fun Optional mirror function to do mirror optimization and save memory. + * \param attr_hint_fun Optional, hint function to output a node that like src, but its attr is same as like. + * \param zero_ops Optional, list of operators that outputs a single zero array. The first one + * must be zeros_like. + * \param copy_op_str Optional, name of the copy operation required to handle duplicates + * on the edge of the graph + * \return A new graph, whose outputs correspond to inputs of xs. + */ +inline Graph MXGradient( + Graph graph, + std::vector ys, + std::vector xs, + std::vector ys_out_grad, + std::function&& inputs)> aggregate_fun = nullptr, + std::function mirror_fun = nullptr, + std::function + attr_hint_fun = nullptr, + std::vector zero_ops = std::vector(), + std::string copy_op_str = std::string()) { + graph.attrs["grad_ys"] = std::make_shared(std::move(ys)); + graph.attrs["grad_xs"] = std::make_shared(std::move(xs)); + graph.attrs["grad_ys_out_grad"] = std::make_shared(std::move(ys_out_grad)); + if (aggregate_fun != nullptr) { + graph.attrs["grad_aggregate_fun"] = std::make_shared(aggregate_fun); + } + if (mirror_fun != nullptr) { + graph.attrs["grad_mirror_fun"] = std::make_shared(mirror_fun); + } + if (attr_hint_fun != nullptr) { + graph.attrs["attr_hint_fun"] = std::make_shared(attr_hint_fun); + } + if (zero_ops.size()) { + graph.attrs["zero_ops"] = std::make_shared(std::move(zero_ops)); + } + if (copy_op_str != std::string()) { + graph.attrs["copy_op"] = std::make_shared(std::move(copy_op_str)); + } + return ApplyPass(std::move(graph), "MXGradient"); +} +} // namespace pass +} // namespace nnvm + #endif // MXNET_EXECUTOR_EXEC_PASS_H_ diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 8302dc133c64..ca2cea093c5d 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -279,7 +279,7 @@ nnvm::Graph GraphExecutor::InitFullGraph(nnvm::Symbol symbol, zero_ops.push_back(nnvm::Op::Get("_zeros")); // take gradient - nnvm::Graph g_grad = nnvm::pass::Gradient( + nnvm::Graph g_grad = nnvm::pass::MXGradient( g, symbol.outputs, xs, head_grad_entry_, AggregateGradient, need_mirror, nullptr, zero_ops, "_copy"); @@ -325,7 +325,7 @@ void GraphExecutor::Init(nnvm::Symbol symbol, const auto& mutable_nodes = idx.mutable_input_nodes(); size_t arg_top = 0, aux_top = 0; data_entry_.resize(idx.num_node_entries()); - nnvm::ShapeVector arg_shapes; + mxnet::ShapeVector arg_shapes; nnvm::DTypeVector arg_dtypes; StorageTypeVector arg_stypes(idx.num_node_entries(), -1); for (size_t i = 0; i < num_forward_inputs_; ++i) { @@ -367,11 +367,11 @@ void GraphExecutor::Init(nnvm::Symbol symbol, } // expand arg_shapes and arg_dtypes to contain backward inputs - arg_shapes.resize(idx.input_nodes().size(), TShape()); + arg_shapes.resize(idx.input_nodes().size(), mxnet::TShape()); g = InferShape(std::move(g), std::move(arg_shapes), "__shape__"); if (g.GetAttr("shape_num_unknown_nodes") != 0U) { HandleInferShapeError(num_forward_inputs_, g.indexed_graph(), - g.GetAttr("shape")); + g.GetAttr("shape")); } arg_dtypes.resize(idx.input_nodes().size(), -1); @@ -401,7 +401,7 @@ void GraphExecutor::Init(nnvm::Symbol symbol, * shared data arrays are provided. */ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, - const nnvm::ShapeVector& inferred_shapes, + const mxnet::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, @@ -419,7 +419,7 @@ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, for (size_t i = 0; i < num_forward_inputs_; ++i) { const uint32_t nid = idx.input_nodes().at(i); const uint32_t eid = idx.entry_id(nid, 0); - const TShape& inferred_shape = inferred_shapes[eid]; + const mxnet::TShape& inferred_shape = inferred_shapes[eid]; const int inferred_dtype = inferred_dtypes[eid]; const NDArrayStorageType inferred_stype = (NDArrayStorageType) inferred_stypes[eid]; const std::string& arg_name = idx[nid].source->attrs.name; @@ -471,7 +471,7 @@ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, * and shared_exec if available. */ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, - const nnvm::ShapeVector& inferred_shapes, + const mxnet::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, @@ -491,7 +491,7 @@ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, for (size_t i = 0; i < num_forward_inputs_; ++i) { const uint32_t nid = idx.input_nodes().at(i); const uint32_t eid = idx.entry_id(nid, 0); - const TShape& inferred_shape = inferred_shapes[eid]; + const mxnet::TShape& inferred_shape = inferred_shapes[eid]; const int inferred_dtype = inferred_dtypes[eid]; const NDArrayStorageType inferred_stype = (NDArrayStorageType) inferred_stypes[eid]; const std::string& arg_name = idx[nid].source->attrs.name; @@ -635,7 +635,7 @@ void GraphExecutor::FinishInitGraph(nnvm::Symbol symbol, if (vstorage_type[i] != kDefaultStorage) arg_storage_id[i] = kDynamicStorageID; } g.attrs["storage"] = std::make_shared(std::move(arg_storage_id)); - g = nnvm::ApplyPass(g, "PlanMemory"); + g = nnvm::ApplyPass(g, "MXPlanMemory"); } g = DetectInplaceAddTo(g); @@ -694,7 +694,7 @@ void GraphExecutor::Init(nnvm::Symbol symbol, const std::vector& in_arg_ctxes, const std::vector& arg_grad_ctxes, const std::vector& aux_state_ctxes, - const std::unordered_map& arg_shape_map, + const std::unordered_map& arg_shape_map, const std::unordered_map& arg_dtype_map, const std::unordered_map& arg_stype_map, const std::vector& grad_req_types, @@ -714,7 +714,7 @@ void GraphExecutor::Init(nnvm::Symbol symbol, // Initialize arg_shapes and arg_dtypes for shape and type inferences. // It contains all in_args and aux_states' shapes and types in a certain order. const nnvm::IndexedGraph& idx = g.indexed_graph(); - nnvm::ShapeVector arg_shapes(idx.input_nodes().size(), TShape()); + mxnet::ShapeVector arg_shapes(idx.input_nodes().size(), mxnet::TShape()); nnvm::DTypeVector arg_dtypes(idx.input_nodes().size(), -1); StorageTypeVector arg_stypes(idx.input_nodes().size(), kUndefinedStorage); for (size_t i = 0; i < num_forward_inputs_; ++i) { @@ -736,7 +736,7 @@ void GraphExecutor::Init(nnvm::Symbol symbol, g = InferShape(std::move(g), std::move(arg_shapes), "__shape__"); if (g.GetAttr("shape_num_unknown_nodes") != 0U) { HandleInferShapeError(num_forward_inputs_, g.indexed_graph(), - g.GetAttr("shape")); + g.GetAttr("shape")); } g = InferType(std::move(g), std::move(arg_dtypes), "__dtype__"); @@ -754,13 +754,13 @@ void GraphExecutor::Init(nnvm::Symbol symbol, // Create in_args, arg_grads, and aux_states using // the inferred shapes and dtypes. if (nullptr == shared_buffer) { // regular simple bind - InitArguments(idx, g.GetAttr("shape"), + InitArguments(idx, g.GetAttr("shape"), g.GetAttr("dtype"), g.GetAttr("storage_type"), in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes, grad_req_types, in_arg_vec, arg_grad_vec, aux_state_vec); } else { // simple bind using shared data arrays and shared_exec - InitArguments(idx, g.GetAttr("shape"), + InitArguments(idx, g.GetAttr("shape"), g.GetAttr("dtype"), g.GetAttr("storage_type"), in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes, @@ -788,7 +788,7 @@ Executor* GraphExecutor::Reshape(const bool partial_shaping, const bool allow_up_sizing, const Context& default_ctx, const std::map& ctx_map, - const std::unordered_map& + const std::unordered_map& provided_arg_shapes, std::vector* in_args, std::vector* arg_grads, @@ -799,7 +799,7 @@ Executor* GraphExecutor::Reshape(const bool partial_shaping, nnvm::Symbol symbol; symbol.outputs = g.outputs; const nnvm::IndexedGraph& idx = g.indexed_graph(); - nnvm::ShapeVector arg_shapes(idx.input_nodes().size(), TShape()); + mxnet::ShapeVector arg_shapes(idx.input_nodes().size(), mxnet::TShape()); for (size_t i = 0; i < num_forward_inputs_; ++i) { const uint32_t nid = idx.input_nodes().at(i); const std::string& name = idx[nid].source->attrs.name; @@ -811,9 +811,9 @@ Executor* GraphExecutor::Reshape(const bool partial_shaping, g = InferShape(std::move(g), std::move(arg_shapes), "__shape__"); if (g.GetAttr("shape_num_unknown_nodes") != 0U) { HandleInferShapeError(num_forward_inputs_, g.indexed_graph(), - g.GetAttr("shape")); + g.GetAttr("shape")); } - const nnvm::ShapeVector& shape_vec = g.GetAttr("shape"); + const mxnet::ShapeVector& shape_vec = g.GetAttr("shape"); std::vector grad_req_types; size_t grad_top = 0; const size_t num_args = in_arg_map_.size(); @@ -824,7 +824,7 @@ Executor* GraphExecutor::Reshape(const bool partial_shaping, aux_states->reserve(num_aux); for (uint32_t nid : idx.input_nodes()) { std::string name = idx[nid].source->attrs.name; - const TShape& new_shape = shape_vec[idx.entry_id(nid, 0)]; + const mxnet::TShape& new_shape = shape_vec[idx.entry_id(nid, 0)]; if (idx.mutable_input_nodes().count(nid) == 0) { NDArray& arr = in_arg_map_.at(name); auto it = arg_grad_map_.find(name); @@ -927,13 +927,13 @@ Graph GraphExecutor::InitGraph(nnvm::Symbol symbol, // initialize the memory of each entries void GraphExecutor::InitDataEntryMemory(std::vector* shared_pool) { using nnvm::DTypeVector; - using nnvm::ShapeVector; + using mxnet::ShapeVector; using nnvm::StorageVector; // get the graph const auto& idx = graph_.indexed_graph(); // get the storage const auto& vdtype = graph_.GetAttr("dtype"); - const auto& vshape = graph_.GetAttr("shape"); + const auto& vshape = graph_.GetAttr("shape"); const auto& vstorage = graph_.GetAttr("storage_id"); const auto& vstorage_type = graph_.GetAttr("storage_type"); const auto& vctx = graph_.GetAttr("context"); @@ -1036,7 +1036,7 @@ void GraphExecutor::InitDataEntryMemory(std::vector* shared_pool) { size_t nword = (bytes + 3) / 4; CHECK_LE(nword, std::numeric_limits::max()); // allocate float arrays - TShape shape{static_cast(nword)}; + mxnet::TShape shape{static_cast(nword)}; // TODO(junwu): adding delay_alloc=true to create nd // is a temporary solution. NDArray nd(shape, ctx, true); @@ -1467,7 +1467,7 @@ GraphExecutor::CachedSegOpr GraphExecutor::CreateCachedSegOpr(size_t topo_start, // Infer shapes, dtypes, stypes, contexts for the forward graph static nnvm::Graph InferForwardAttrs(nnvm::Graph g, - nnvm::ShapeVector arg_shapes, + mxnet::ShapeVector arg_shapes, nnvm::DTypeVector arg_dtypes, StorageTypeVector arg_stypes, const Context& default_ctx, @@ -1481,7 +1481,7 @@ static nnvm::Graph InferForwardAttrs(nnvm::Graph g, g = InferShape(std::move(g), std::move(arg_shapes), "__shape__"); if (g.GetAttr("shape_num_unknown_nodes") != 0U) { HandleInferShapeError(num_forward_inputs, indexed_graph, - g.GetAttr("shape")); + g.GetAttr("shape")); } g = InferType(std::move(g), std::move(arg_dtypes), "__dtype__"); if (g.GetAttr("dtype_num_unknown_nodes") != 0U) { @@ -1500,7 +1500,7 @@ static nnvm::Graph InferForwardAttrs(nnvm::Graph g, // This is a common function for bind and simple_bind flows. static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src, const std::string& prop_name, - const nnvm::ShapeVector& arg_shapes, + const mxnet::ShapeVector& arg_shapes, const nnvm::DTypeVector& arg_dtypes, const StorageTypeVector& arg_stypes, const Context& default_ctx, @@ -1532,7 +1532,8 @@ static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src, // This is for simple_bind flow. static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src, const std::string& prop_name, - const std::unordered_map& arg_shape_map, + const std::unordered_map + & arg_shape_map, const std::unordered_map& arg_dtype_map, const std::unordered_map& arg_stype_map, const Context& default_ctx, @@ -1540,7 +1541,7 @@ static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src, const std::vector& in_arg_ctxes, const std::vector& aux_state_ctxes) { const std::vector input_names = src.ListInputNames(Symbol::kAll); - nnvm::ShapeVector arg_shapes(input_names.size(), TShape()); + mxnet::ShapeVector arg_shapes(input_names.size(), mxnet::TShape()); nnvm::DTypeVector arg_dtypes(input_names.size(), -1); StorageTypeVector arg_stypes(input_names.size(), kUndefinedStorage); for (size_t i = 0; i < input_names.size(); ++i) { @@ -1574,7 +1575,7 @@ static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src, const std::vector aux_names = src.ListInputNames(nnvm::Symbol::kAuxiliaryStates); CHECK_EQ(arg_names.size(), in_args->size()); CHECK_EQ(aux_names.size(), aux_states.size()); - nnvm::ShapeVector arg_shapes; // all input shapes + mxnet::ShapeVector arg_shapes; // all input shapes arg_shapes.reserve(input_names.size()); nnvm::DTypeVector arg_dtypes; // all input dtypes arg_dtypes.reserve(input_names.size()); @@ -1629,7 +1630,7 @@ Executor *Executor::SimpleBind(nnvm::Symbol symbol, const std::vector& in_arg_ctxes, const std::vector& arg_grad_ctxes, const std::vector& aux_state_ctxes, - const std::unordered_map& arg_shape_map, + const std::unordered_map& arg_shape_map, const std::unordered_map& arg_dtype_map, const std::unordered_map& arg_stype_map, const std::vector& grad_req_types, diff --git a/src/executor/graph_executor.h b/src/executor/graph_executor.h index c899a6f5b463..ed49e5bc8bc9 100644 --- a/src/executor/graph_executor.h +++ b/src/executor/graph_executor.h @@ -94,7 +94,7 @@ class GraphExecutor : public Executor { const std::vector& in_arg_ctxes, const std::vector& arg_grad_ctxes, const std::vector& aux_state_ctxes, - const std::unordered_map& arg_shape_map, + const std::unordered_map& arg_shape_map, const std::unordered_map& arg_dtype_map, const std::unordered_map& arg_stype_map, const std::vector& grad_req_types, @@ -111,7 +111,7 @@ class GraphExecutor : public Executor { const bool allow_up_sizing, const Context& default_ctx, const std::map& ctx_map, - const std::unordered_map& + const std::unordered_map& provided_arg_shapes, std::vector* in_args, std::vector* arg_grads, @@ -153,7 +153,7 @@ class GraphExecutor : public Executor { }; // Initialize in_args, arg_grads, and aux_states void InitArguments(const nnvm::IndexedGraph& idx, - const nnvm::ShapeVector& inferred_shapes, + const mxnet::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, @@ -166,7 +166,7 @@ class GraphExecutor : public Executor { // Initialize in_args, arg_grads and aux_states with // shared_buffer and shared_exec virtual void InitArguments(const nnvm::IndexedGraph& idx, - const nnvm::ShapeVector& inferred_shapes, + const mxnet::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, diff --git a/src/executor/infer_graph_attr_pass.cc b/src/executor/infer_graph_attr_pass.cc index c14482affbde..af8094ad92af 100644 --- a/src/executor/infer_graph_attr_pass.cc +++ b/src/executor/infer_graph_attr_pass.cc @@ -322,8 +322,274 @@ nnvm::Graph InferAttr(nnvm::Graph &&ret, return ret; } +template +nnvm::Graph InferShapeAttr(nnvm::Graph &&ret, + const nnvm::TShape empty_val, + const char* infer_name, + const char* input_name, + const char* attr_key_name, + const char* attr_name, + const char* unknown_name, + IsNone fis_none, + FDefault fdefault, + bool bwd_identity_assign, + const char* dispatch_mode_name, + const DispatchMode default_mode_val = DispatchMode::kUndefined) { + using nnvm::IndexedGraph; + using nnvm::Op; + using AttrType = nnvm::TShape; + using FInferType = nnvm::FInferShape; + using AttrVector = std::vector; + using NodeAttrVector = std::vector; + using dmlc::any; + const IndexedGraph& idx = ret.indexed_graph(); + static auto& finfer_shape = + Op::GetAttr(infer_name); + static auto& is_backward = + Op::GetAttr("TIsBackward"); + // gradient function, used to get node correspondence. + static auto& fgrad = + Op::GetAttr("FGradient"); + // reshape shape vector + AttrVector rshape; + // dispatch mode vector + DispatchModeVector dispatch_modes; + if (ret.attrs.count(attr_name) != 0) { + rshape = ret.MoveCopyAttr(attr_name); + } else { + rshape.resize(idx.num_node_entries(), empty_val); + } + + if (ret.attrs.count(input_name) != 0) { + const AttrVector& shape_args = ret.GetAttr(input_name); + CHECK_LE(shape_args.size(), idx.input_nodes().size()) + << "More provided " << attr_name << "s than number of arguments."; + for (size_t i = 0; i < shape_args.size(); ++i) { + rshape[idx.entry_id(idx.input_nodes()[i], 0)] = shape_args[i]; + } + } + + // get the shape hints + std::string shape_hints_key = std::string(attr_name) + "_hints"; + if (ret.attrs.count(shape_hints_key)) { + nnvm::NodeEntryMap shape_hints = + ret.GetAttr>(shape_hints_key); + for (const auto& kv : shape_hints) { + nnvm::NodeEntry e = kv.first; + if (idx.exist(e.node.get())) { + rshape[idx.entry_id(kv.first)] = kv.second; + } + } + } + + std::string shape_attr_key; + if (ret.attrs.count(attr_key_name) != 0) { + shape_attr_key = ret.GetAttr(attr_key_name); + // erase the provided arguments + ret.attrs.erase(attr_key_name); + } + + // limit inference to part of the graph + uint32_t node_start = 0, node_end = idx.num_nodes(); + if (ret.attrs.count("node_range")) { + const auto& range = ret.GetAttr >("node_range"); + node_start = range.first; + node_end = range.second; + CHECK_GE(node_start, 0); + CHECK_LE(node_end, idx.num_nodes()); + ret.attrs.erase("node_range"); + } + uint32_t entry_start = 0, entry_end = idx.num_node_entries(); + if (ret.attrs.count("entry_range")) { + const auto& range = ret.GetAttr >("entry_range"); + entry_start = range.first; + entry_end = range.second; + CHECK_GE(entry_start, 0); + CHECK_LE(entry_end, idx.num_node_entries()); + ret.attrs.erase("entry_range"); + } + // populate the node attribute vector + if (dispatch_mode_name != nullptr) { + if (ret.attrs.count(dispatch_mode_name) != 0) { + dispatch_modes = ret.MoveCopyAttr(dispatch_mode_name); + } else { + LOG(FATAL) << "Node attribute " << dispatch_mode_name << " does not exist in the graph"; + } + } + + // Temp space for shape inference. + std::vector ishape, oshape; + // whether a shape is dynamic + std::vector is_dynamic(rshape.size(), 0); + // inference step function for nid + auto infer_step = [&](uint32_t nid, bool last_iter) { + const auto& inode = idx[nid]; + const std::string name = inode.source->attrs.name; + const uint32_t num_inputs = inode.inputs.size(); + const uint32_t num_outputs = inode.source->num_outputs(); + if (inode.source->is_variable()) { + // Variable node. No operator. Only one output entry. + CHECK(inode.source->op() == nullptr); + CHECK_EQ(num_outputs, 1U); + const uint32_t out_ent_id = idx.entry_id(nid, 0); + if (shape_attr_key.length() != 0 && fis_none(rshape[out_ent_id])) { + auto it = inode.source->attrs.dict.find(shape_attr_key); + if (it != inode.source->attrs.dict.end()) { + std::istringstream is(it->second); + CHECK(is >> rshape[out_ent_id]) << "Invalid attribute"; + } + } + // assign a default value to node attribute + if (dispatch_mode_name != nullptr) { + op::dispatch_mode_assign(&dispatch_modes[nid], default_mode_val); + } + } else if (is_backward.get(inode.source->op(), false) && + inode.control_deps.size() && bwd_identity_assign) { + CHECK(dispatch_mode_name == nullptr) + << "Backward inference for node attributes is not available"; + CHECK_GE(inode.control_deps.size(), 1U) + << "BackwardOp need to have control_deps to its forward op"; + const IndexedGraph::Node& fnode = idx[inode.control_deps[0]]; + nnvm::NodePtr fwd_ptr = inode.source->control_deps[0]; + CHECK(fwd_ptr->op() != nullptr) << "Forward op cannot be a variable"; + // use gradient function to find out the correspondence. + std::vector ograd(fwd_ptr->num_outputs()); + for (size_t i = 0; i < ograd.size(); ++i) { + ograd[i].index = static_cast(i); + } + // input gradient list + auto igrad = fgrad[fwd_ptr->op()](fwd_ptr, ograd); + const nnvm::Node* igrad_node = nullptr; + // Input gradient assignement + for (size_t i = 0; i < igrad.size(); ++i) { + if (igrad[i].node->op() == inode.source->op()) { + uint32_t eid = idx.entry_id(nid, igrad[i].index); + if (fis_none(rshape[eid])) { + rshape[eid] = rshape[idx.entry_id(fnode.inputs[i])]; + } else if (!fis_none(rshape[idx.entry_id(fnode.inputs[i])])) { + // Need to skip empty forward shape, because it may not be + // available now and it is possible to infer the forward + // shape in one of the next a few passes + CHECK_EQ(rshape[eid], rshape[idx.entry_id(fnode.inputs[i])]) + << "Backward shape inconsistent with the forward shape"; + } + if (igrad_node == nullptr) { + igrad_node = igrad[i].node.get(); + } else { + CHECK(igrad_node == igrad[i].node.get()); + } + } + } + // out grad entries + CHECK(igrad_node != nullptr) + << "Cannot find matching backward op for " << inode.source->attrs.name; + for (size_t i = 0; i < igrad_node->inputs.size(); ++i) { + const nnvm::NodeEntry& e = igrad_node->inputs[i]; + if (e.node == nullptr) { + uint32_t eid = idx.entry_id(inode.inputs[i]); + if (fis_none(rshape[eid])) { + rshape[eid] = rshape[idx.entry_id(inode.control_deps[0], e.index)]; + } + } + } + } else { + DispatchMode* dispatch_mode = nullptr; + bool forward_known = true; + // Forward operator inference. + ishape.resize(num_inputs, empty_val); + bool is_input_dynamic_shape = false; + for (uint32_t i = 0; i < ishape.size(); ++i) { + ishape[i] = rshape[idx.entry_id(inode.inputs[i])]; + if (ishape[i].ndim() == 0 && is_dynamic[idx.entry_id(inode.inputs[i])]) { + is_input_dynamic_shape = true; + } + if (fis_none(ishape[i])) forward_known = false; + } + oshape.resize(num_outputs, empty_val); + for (uint32_t i = 0; i < oshape.size(); ++i) { + oshape[i] = rshape[idx.entry_id(nid, i)]; + if (fis_none(oshape[i])) forward_known = false; + } + if (dispatch_mode_name != nullptr) { + dispatch_mode = &dispatch_modes[nid]; + if (dispatch_modes[nid] == DispatchMode::kUndefined) forward_known = false; + } + auto finfer = finfer_shape.get(inode.source->op(), fdefault); + if (finfer == nullptr || is_input_dynamic_shape) { + for (uint32_t i = 0; i < oshape.size(); ++i) { + if (oshape[i].ndim() == 0) { + is_dynamic[idx.entry_id(nid, i)] = 1; + } + } + } else if (!forward_known) { + if (finfer != nullptr) { + // Call inference function of the operator. + try { + forward_known = ApplyOpInferAttr(ret, finfer, inode.source->attrs, + nid, &ishape, &oshape, dispatch_mode); + } catch (const std::exception& e) { + throw dmlc::Error("Error in operator " + inode.source->attrs.name + ": " + e.what()); + } + } else { + CHECK(!last_iter) + << "Attribute " << infer_name + << " is not registed by op " << inode.source->op()->name + << " we are not able to complete the inference because of this"; + } + } + // Save to the result map. + for (uint32_t i = 0; i < num_inputs; ++i) { + rshape[idx.entry_id(inode.inputs[i])] = ishape[i]; + } + for (uint32_t i = 0; i < num_outputs; ++i) { + rshape[idx.entry_id(nid, i)] = oshape[i]; + } + } + }; + + size_t last_num_unknown; + size_t num_unknown_dispatch_mode = dispatch_mode_name ? node_end - node_start : 0; + size_t num_unknown_entry_attr = entry_end - entry_start; + size_t num_unknown = num_unknown_entry_attr + num_unknown_dispatch_mode; + int i = 0; + do { + if (i % 2 == 0) { + for (uint32_t nid = node_start; nid < node_end; ++nid) { + infer_step(nid, false); + } + } else { + // backward inference + for (uint32_t i = node_end; i != node_start; --i) { + infer_step(i - 1, false); + } + } + last_num_unknown = num_unknown; + num_unknown = 0; + for (size_t j = entry_start; j < entry_end; ++j) { + if (fis_none(rshape[j])) { + ++num_unknown; + } + } + if (dispatch_mode_name) { + for (size_t i = node_start; i < node_end; i++) { + if (dispatch_modes[i] == DispatchMode::kUndefined) ++num_unknown; + } + } + ++i; + } while (num_unknown > 0 && last_num_unknown > num_unknown); + // set the shapes + ret.attrs[attr_name] = std::make_shared(std::move(rshape)); + // set the shapes + if (dispatch_mode_name) { + ret.attrs[dispatch_mode_name] = std::make_shared(std::move(dispatch_modes)); + } + // number of nodes who knows the shape. + ret.attrs[unknown_name] = std::make_shared(num_unknown); + return ret; +} + nnvm::Graph InferShape(nnvm::Graph&& graph, - nnvm::ShapeVector&& shape_inputs, + mxnet::ShapeVector&& shape_inputs, const std::string& shape_attr_key) { using dmlc::any; if (shape_inputs.size() != 0) { @@ -332,11 +598,11 @@ nnvm::Graph InferShape(nnvm::Graph&& graph, if (shape_attr_key.length() != 0) { graph.attrs["shape_attr_key"] = std::make_shared(shape_attr_key); } - return InferAttr( - std::move(graph), nnvm::TShape(), + return InferAttr( + std::move(graph), mxnet::TShape(), "FInferShape", "shape_inputs", "shape_attr_key", "shape", "shape_num_unknown_nodes", - [](const nnvm::TShape& s) { return s.ndim() == 0 || s.Size() == 0; }, + [](const mxnet::TShape& s) { return s.ndim() == 0 || s.Size() == 0; }, nullptr, true, nullptr); } diff --git a/src/executor/tensorrt_pass.cc b/src/executor/tensorrt_pass.cc index 762dc0de9db5..f847d59a1298 100644 --- a/src/executor/tensorrt_pass.cc +++ b/src/executor/tensorrt_pass.cc @@ -343,17 +343,17 @@ Graph UpdateSubgraphAttrs(Graph&& subgraph, const Graph& g, const auto& idx = g.indexed_graph(); const auto& sub_idx = subgraph.indexed_graph(); - const auto& shape = g.GetAttr("shape"); + const auto& shape = g.GetAttr("shape"); const auto& dtype = g.GetAttr("dtype"); const auto& storage_type = g.GetAttr("storage_type"); - const auto& shape_inputs = g.GetAttr("shape_inputs"); + const auto& shape_inputs = g.GetAttr("shape_inputs"); const auto& dtype_inputs = g.GetAttr("dtype_inputs"); const auto& storage_type_inputs = g.GetAttr("storage_type_inputs"); - nnvm::ShapeVector sub_shape(sub_idx.num_node_entries()); + mxnet::ShapeVector sub_shape(sub_idx.num_node_entries()); nnvm::DTypeVector sub_dtype(sub_idx.num_node_entries()); StorageTypeVector sub_storage_type(sub_idx.num_node_entries()); - nnvm::ShapeVector sub_shape_inputs(sub_idx.input_nodes().size()); + mxnet::ShapeVector sub_shape_inputs(sub_idx.input_nodes().size()); nnvm::DTypeVector sub_dtype_inputs(sub_idx.input_nodes().size()); StorageTypeVector sub_storage_type_inputs(sub_idx.input_nodes().size()); diff --git a/src/executor/trt_graph_executor.cc b/src/executor/trt_graph_executor.cc index 85ce16885c88..c923922d5184 100644 --- a/src/executor/trt_graph_executor.cc +++ b/src/executor/trt_graph_executor.cc @@ -60,7 +60,7 @@ void TrtGraphExecutor::Init(nnvm::Symbol symbol, std::vector *in_arg_ctxes, std::vector *arg_grad_ctxes, std::vector *aux_state_ctxes, - std::unordered_map *arg_shape_map, + std::unordered_map *arg_shape_map, std::unordered_map *arg_dtype_map, std::unordered_map *arg_stype_map, std::vector *grad_req_types, @@ -95,7 +95,7 @@ void TrtGraphExecutor::Init(nnvm::Symbol symbol, // Initialize arg_shapes and arg_dtypes for shape and type inferences. // It contains all in_args and aux_states' shapes and types in a certain order. const nnvm::IndexedGraph& idx = g.indexed_graph(); - nnvm::ShapeVector arg_shapes(idx.input_nodes().size(), TShape()); + mxnet::ShapeVector arg_shapes(idx.input_nodes().size(), mxnet::TShape()); nnvm::DTypeVector arg_dtypes(idx.input_nodes().size(), -1); StorageTypeVector arg_stypes(idx.input_nodes().size(), kUndefinedStorage); for (size_t i = 0; i < num_forward_inputs_; ++i) { @@ -117,7 +117,7 @@ void TrtGraphExecutor::Init(nnvm::Symbol symbol, g = InferShape(std::move(g), std::move(arg_shapes), "__shape__"); if (g.GetAttr("shape_num_unknown_nodes") != 0U) { HandleInferShapeError(num_forward_inputs_, g.indexed_graph(), - g.GetAttr("shape")); + g.GetAttr("shape")); } g = InferType(std::move(g), std::move(arg_dtypes), "__dtype__"); @@ -142,7 +142,7 @@ void TrtGraphExecutor::Init(nnvm::Symbol symbol, } } - InitArguments(g.indexed_graph(), g.GetAttr("shape"), + InitArguments(g.indexed_graph(), g.GetAttr("shape"), g.GetAttr("dtype"), g.GetAttr("storage_type"), *in_arg_ctxes, *arg_grad_ctxes, *aux_state_ctxes, @@ -165,7 +165,7 @@ void TrtGraphExecutor::Init(nnvm::Symbol symbol, * and shared_exec if available. */ void TrtGraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, - const nnvm::ShapeVector& inferred_shapes, + const mxnet::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, @@ -185,7 +185,7 @@ void TrtGraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, for (size_t i = 0; i < num_forward_inputs_; ++i) { const uint32_t nid = idx.input_nodes().at(i); const uint32_t eid = idx.entry_id(nid, 0); - const TShape& inferred_shape = inferred_shapes[eid]; + const mxnet::TShape& inferred_shape = inferred_shapes[eid]; const int inferred_dtype = inferred_dtypes[eid]; const auto inferred_stype = (NDArrayStorageType) inferred_stypes[eid]; const std::string& arg_name = idx[nid].source->attrs.name; @@ -319,7 +319,7 @@ Graph TrtGraphExecutor::ReinitGraph(Graph&& g, const Context &default_ctx, std::vector *arg_grad_ctxes, std::vector *aux_state_ctxes, std::vector *grad_req_types, - std::unordered_map *arg_shape_map, + std::unordered_map *arg_shape_map, std::unordered_map *arg_dtype_map, std::unordered_map *arg_stype_map, std::unordered_map *params_map) { @@ -356,7 +356,7 @@ Graph TrtGraphExecutor::ReinitGraph(Graph&& g, const Context &default_ctx, num_forward_nodes_ = std::max( num_forward_nodes_, static_cast(idx.outputs()[i].node_id + 1)); } - nnvm::ShapeVector arg_shapes(idx.input_nodes().size(), TShape()); + mxnet::ShapeVector arg_shapes(idx.input_nodes().size(), mxnet::TShape()); nnvm::DTypeVector arg_dtypes(idx.input_nodes().size(), -1); StorageTypeVector arg_stypes(idx.input_nodes().size(), kUndefinedStorage); for (size_t i = 0; i < num_forward_inputs_; ++i) { @@ -378,7 +378,7 @@ Graph TrtGraphExecutor::ReinitGraph(Graph&& g, const Context &default_ctx, g = InferShape(std::move(g), std::move(arg_shapes), "__shape__"); if (g.GetAttr("shape_num_unknown_nodes") != 0U) { HandleInferShapeError(num_forward_inputs_, g.indexed_graph(), - g.GetAttr("shape")); + g.GetAttr("shape")); } g = InferType(std::move(g), std::move(arg_dtypes), "__dtype__"); @@ -415,7 +415,8 @@ Executor *TrtGraphExecutor::TensorRTBind(nnvm::Symbol symbol, std::vector *in_arg_ctxes, std::vector *arg_grad_ctxes, std::vector *aux_state_ctxes, - std::unordered_map *arg_shape_map, + std::unordered_map + *arg_shape_map, std::unordered_map *arg_dtype_map, std::unordered_map *arg_stype_map, std::vector *grad_req_types, diff --git a/src/executor/trt_graph_executor.h b/src/executor/trt_graph_executor.h index 96ac4426270a..a4ec5bf657ae 100644 --- a/src/executor/trt_graph_executor.h +++ b/src/executor/trt_graph_executor.h @@ -40,7 +40,7 @@ class TrtGraphExecutor : public GraphExecutor { std::vector *in_arg_ctxes, std::vector* arg_grad_ctxes, std::vector* aux_state_ctxes, - std::unordered_map* arg_shape_map, + std::unordered_map* arg_shape_map, std::unordered_map* arg_dtype_map, std::unordered_map* arg_stype_map, std::vector* grad_req_types, @@ -58,7 +58,7 @@ class TrtGraphExecutor : public GraphExecutor { std::vector *in_arg_ctxes, std::vector *arg_grad_ctxes, std::vector *aux_state_ctxes, - std::unordered_map *arg_shape_map, + std::unordered_map *arg_shape_map, std::unordered_map *arg_dtype_map, std::unordered_map *arg_stype_map, std::vector *grad_req_types, @@ -81,13 +81,13 @@ class TrtGraphExecutor : public GraphExecutor { std::vector *arg_grad_ctxes, std::vector *aux_state_ctxes, std::vector *grad_req_types, - std::unordered_map *arg_shape_map, + std::unordered_map *arg_shape_map, std::unordered_map *arg_dtype_map, std::unordered_map *arg_stype_map, std::unordered_map *params_map); void InitArguments(const nnvm::IndexedGraph& idx, - const nnvm::ShapeVector& inferred_shapes, + const mxnet::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 58ec4e65b846..9db235bca532 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -175,7 +175,7 @@ CachedOp::CachedOp( CHECK_GT(xs.size(), 0) << "There are no inputs in computation graph that require gradients."; - grad_graph_ = pass::Gradient( + grad_graph_ = pass::MXGradient( fwd_graph_, fwd_graph_.outputs, xs, ograd_entries_, exec::AggregateGradient, nullptr, nullptr, zero_ops, "_copy"); @@ -285,12 +285,20 @@ bool CachedOp::SetForwardGraph( } bool match = true; - match &= CheckAndInferShape(&g, std::move(shape_inputs), true); + bool contain_dynamic_shape = false; + match &= CheckAndInferShape(&g, std::move(shape_inputs), true, + {0, 0}, {0, 0}, &contain_dynamic_shape); match &= CheckAndInferType(&g, std::move(dtype_inputs), true); exec::DevMaskVector dev_mask(g.indexed_graph().num_nodes(), inputs[0]->ctx().dev_mask()); match &= CheckAndInferStorageType(&g, std::move(dev_mask), std::move(storage_type_inputs), true); + // When dynmaic shape exists, it is not feasible to plan memory ahead of time + if (contain_dynamic_shape) { + g.attrs.erase("forward_mem_plan"); + g.attrs.erase("full_mem_plan"); + return false; + } if (!match) { g.attrs.erase("forward_mem_plan"); g.attrs.erase("full_mem_plan"); @@ -397,8 +405,8 @@ bool CachedOp::SetBackwardGraph( g.attrs["backward_ref_count"] = std::make_shared(std::move(ref_count)); } - auto shapes = info->fwd_graph.GetAttr("shape"); - shapes.resize(idx.num_node_entries(), TShape()); + auto shapes = info->fwd_graph.GetAttr("shape"); + shapes.resize(idx.num_node_entries(), mxnet::TShape()); auto dtypes = info->fwd_graph.GetAttr("dtype"); dtypes.resize(idx.num_node_entries(), -1); auto stypes = info->fwd_graph.GetAttr("storage_type"); @@ -616,7 +624,7 @@ void CachedOp::StaticRunOps( const auto& op_execs = state.execs; std::vector ndinputs, ndoutputs; - nnvm::ShapeVector arg_shapes; + mxnet::ShapeVector arg_shapes; nnvm::DTypeVector arg_dtypes; std::vector req; @@ -731,7 +739,7 @@ OpStatePtr CachedOp::StaticForward( } const auto& dtypes = g.GetAttr("dtype"); - const auto& shapes = g.GetAttr("shape"); + const auto& shapes = g.GetAttr("shape"); const auto& stypes = g.GetAttr("storage_type"); for (size_t i = 0; i < outputs.size(); ++i) { @@ -808,7 +816,7 @@ OpStatePtr CachedOp::DynamicForward( mem_plan, arrays, &array_reqs); const auto& dtypes = g.GetAttr("dtype"); - const auto& shapes = g.GetAttr("shape"); + const auto& shapes = g.GetAttr("shape"); const auto& stypes = g.GetAttr("storage_type"); for (size_t i = 0; i < outputs.size(); ++i) { @@ -1222,7 +1230,7 @@ void CachedOpBackward(const OpStatePtr& state_ptr, OpStatePtr CreateCachedOpState(const NodeAttrs& attrs, Context ctx, - const std::vector& in_shapes, + const mxnet::ShapeVector& in_shapes, const std::vector& in_types) { const CachedOpPtr& op = nnvm::get(attrs.parsed); return OpStatePtr::Create(op); @@ -1327,10 +1335,10 @@ NNVM_REGISTER_OP(_CachedOp) return op->ListForwardOutputNames(); }) .set_attr("FCreateOpState", CreateCachedOpState) -.set_attr("FInferShape", +.set_attr("FInferShape", [](const nnvm::NodeAttrs& attrs, - std::vector *in_shapes, - std::vector *out_shapes) { + mxnet::ShapeVector *in_shapes, + mxnet::ShapeVector *out_shapes) { const CachedOpPtr& op = nnvm::get(attrs.parsed); return op::DefaultSubgraphOpShapeHelper(op->GetForwardSym(), in_shapes, out_shapes); }) diff --git a/src/imperative/cached_op.h b/src/imperative/cached_op.h index 59a793ee1b65..3b173c8654a4 100644 --- a/src/imperative/cached_op.h +++ b/src/imperative/cached_op.h @@ -35,6 +35,7 @@ struct CachedOpConfig : public dmlc::Parameter { uint32_t backward_bulk_size; bool static_alloc; bool static_shape; + bool is_dynamic; nnvm::Tuple data_indices; nnvm::Tuple param_indices; std::string subgraph; @@ -66,6 +67,9 @@ struct CachedOpConfig : public dmlc::Parameter { DMLC_DECLARE_FIELD(subgraph) .set_default(std::string("")) .describe("JSON string of a subgraph."); + DMLC_DECLARE_FIELD(is_dynamic) + .set_default(false) + .describe("Whether the graph contains dynamic shape operators."); } }; diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index a381b2384113..8d1f65518565 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -351,7 +351,7 @@ std::vector Imperative::Backward( << "There are no inputs in computation graph that require gradients."; } - Graph g_graph = pass::Gradient( + Graph g_graph = pass::MXGradient( graph, graph.outputs, xs, ograd_entries, exec::AggregateGradient, nullptr, nullptr, zero_ops, "_copy"); @@ -479,7 +479,7 @@ std::vector Imperative::Backward( array_reqs[eid] = x_reqs[i - num_forward_outputs]; } - const auto& shapes = graph.GetAttr("shape"); + const auto& shapes = graph.GetAttr("shape"); const auto& dtypes = graph.GetAttr("dtype"); const auto& stypes = graph.GetAttr("storage_type"); const auto& dispatch_modes = graph.GetAttr("dispatch_mode"); diff --git a/src/imperative/imperative_utils.h b/src/imperative/imperative_utils.h index 6446c37627a9..5eecfe8c6f23 100644 --- a/src/imperative/imperative_utils.h +++ b/src/imperative/imperative_utils.h @@ -100,18 +100,18 @@ inline void SetShapeType(const Context& ctx, const std::vector& inputs, const std::vector& outputs, DispatchMode* dispatch_mode) { - static auto& infershape = nnvm::Op::GetAttr("FInferShape"); + static auto& infershape = nnvm::Op::GetAttr("FInferShape"); static auto& infertype = nnvm::Op::GetAttr("FInferType"); static auto& inferstorage = nnvm::Op::GetAttr("FInferStorageType"); MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); // infer shape - std::vector& in_shapes = ret->arg_shapes; + mxnet::ShapeVector& in_shapes = ret->arg_shapes; in_shapes.clear(); in_shapes.reserve(inputs.size()); for (auto& i : inputs) { in_shapes.push_back(i->shape()); } - std::vector& out_shapes = ret->out_shapes; + mxnet::ShapeVector& out_shapes = ret->out_shapes; out_shapes.clear(); out_shapes.reserve(outputs.size()); for (auto& i : outputs) { @@ -179,7 +179,7 @@ inline void SetShapeType(const Context& ctx, for (size_t i = 0; i < outputs.size(); ++i) { NDArrayStorageType storage_type = static_cast(out_storage_types[i]); - if (outputs[i]->is_none()) { + if (outputs[i]->is_none() || outputs[i]->shape().ndim() == 0) { if (is_dynamic_shape_existing) { // once there is dynamic shape somewhere, we could not pre-determine the shape. *outputs[i] = NDArray(ctx, out_types[i]); @@ -448,7 +448,7 @@ inline void PushFComputeEx(const FComputeEx& fn, }; if (exec_type == ExecType::kCrossDeviceCopy) { - run(RunContext{ctx, nullptr}); + run(RunContext{ctx, nullptr, nullptr, false}); } else { CHECK(exec_type == ExecType::kSync); Engine::Get()->PushSync(run, ctx, read_vars, write_vars, FnProperty::kNormal, @@ -498,7 +498,7 @@ inline void PushOperator(const OpStatePtr& state, // For operators with subgraphs, we need to invoke them in the main thread // instead of the threaded engine. if (exec_type == ExecType::kSubgraphExec) { - RunContext rctx{ctx, nullptr}; + RunContext rctx{ctx, nullptr, nullptr, false}; run(rctx, engine::CallbackOnComplete()); } else if (exec_type == ExecType::kSync) { Engine::Get()->PushSync( @@ -546,7 +546,7 @@ inline void PushOperator(const OpStatePtr& state, }; if (exec_type == ExecType::kSubgraphExec) { - RunContext rctx{ctx, nullptr}; + RunContext rctx{ctx, nullptr, nullptr, false}; run(rctx, engine::CallbackOnComplete()); } else if (exec_type == ExecType::kSync) { Engine::Get()->PushSync( @@ -563,17 +563,21 @@ inline void PushOperator(const OpStatePtr& state, } } -inline bool CheckAndInferShape(nnvm::Graph* p_g, nnvm::ShapeVector&& shapes, +inline bool CheckAndInferShape(nnvm::Graph* p_g, mxnet::ShapeVector&& shapes, bool use_inputs, std::pair node_range = {0, 0}, - std::pair entry_range = {0, 0}) { + std::pair entry_range = {0, 0}, + bool *contain_unknown = nullptr) { using namespace nnvm; + if (contain_unknown != nullptr) { + *contain_unknown = false; + } nnvm::Graph& g = *p_g; if (use_inputs) { if (g.attrs.count("shape_inputs") && - g.GetAttr("shape_inputs") == shapes) return true; + g.GetAttr("shape_inputs") == shapes) return true; } else if (g.attrs.count("shape")) { - const auto& prev_shapes = g.GetAttr("shape"); + const auto& prev_shapes = g.GetAttr("shape"); CHECK_EQ(prev_shapes.size(), shapes.size()); bool match = true; for (size_t i = 0; i < shapes.size(); ++i) { @@ -601,8 +605,11 @@ inline bool CheckAndInferShape(nnvm::Graph* p_g, nnvm::ShapeVector&& shapes, g.attrs["shape"] = std::make_shared(std::move(shapes)); g = exec::InferShape(std::move(g)); } - CHECK_EQ(g.GetAttr("shape_num_unknown_nodes"), 0U); - + if (contain_unknown == nullptr) { + CHECK_EQ(g.GetAttr("shape_num_unknown_nodes"), 0U); + } else { + *contain_unknown = g.GetAttr("shape_num_unknown_nodes") != 0U; + } return false; } @@ -766,11 +773,11 @@ inline MemoryPlanVector PlanMemory( } g.attrs["ref_count"] = std::make_shared(ref_count); g.attrs["storage"] = std::make_shared(std::move(storage)); - g = nnvm::ApplyPass(g, "PlanMemory"); + g = nnvm::ApplyPass(g, "MXPlanMemory"); if (detect_inplace_addto) g = exec::DetectInplaceAddTo(g); const auto& dtypes = g.GetAttr("dtype"); - const auto& shapes = g.GetAttr("shape"); + const auto& shapes = g.GetAttr("shape"); const auto& storage_inplace = g.GetAttr >("storage_inplace_index"); const auto& storage_ids = g.GetAttr("storage_id"); uint32_t entry_start = entry_range.first; @@ -811,7 +818,7 @@ inline std::multimap AllocateMemory( std::multimap&& pool = std::multimap()) { using namespace nnvm; const auto& dtypes = g.GetAttr("dtype"); - const auto& shapes = g.GetAttr("shape"); + const auto& shapes = g.GetAttr("shape"); const auto& stypes = g.GetAttr("storage_type"); std::multimap new_pool; @@ -833,7 +840,7 @@ inline std::multimap AllocateMemory( new_pool.insert(*iter); pool.erase(iter); } else { - NDArray buff(TShape({static_cast(mem_plan[i].size)}), + NDArray buff(mxnet::TShape({static_cast(mem_plan[i].size)}), default_ctx, true, mshadow::kUint8); *arrays[i] = buff.AsArray(shapes[i], dtypes[i]); new_pool.insert({mem_plan[i].size, buff}); diff --git a/src/io/image_aug_default.cc b/src/io/image_aug_default.cc index cd06de2b2ad1..5fb0b0f21e7e 100644 --- a/src/io/image_aug_default.cc +++ b/src/io/image_aug_default.cc @@ -96,7 +96,7 @@ struct DefaultImageAugmentParam : public dmlc::Parameter #include #include -#include #include #include @@ -189,7 +188,7 @@ void Imdecode(const nnvm::NodeAttrs& attrs, size_t len = inputs[0].shape().Size(); CHECK(len > 0) << "Input cannot be an empty buffer"; - TShape oshape(3); + mxnet::TShape oshape(3); oshape[2] = param.flag == 0 ? 1 : 3; if (get_jpeg_size(str_img, len, &oshape[1], &oshape[0])) { } else if (get_png_size(str_img, len, &oshape[1], &oshape[0])) { @@ -229,7 +228,7 @@ void Imread(const nnvm::NodeAttrs& attrs, CHECK(file.good()) << "Failed reading image file: '" << param.filename << "' " << strerror(errno); - TShape oshape(3); + mxnet::TShape oshape(3); oshape[2] = param.flag == 0 ? 1 : 3; if (get_jpeg_size(buff.get(), fsize, &oshape[1], &oshape[0])) { } else if (get_png_size(buff.get(), fsize, &oshape[1], &oshape[0])) { @@ -271,8 +270,8 @@ struct ResizeParam : public dmlc::Parameter { DMLC_REGISTER_PARAMETER(ResizeParam); inline bool ResizeShape(const nnvm::NodeAttrs& attrs, - std::vector *ishape, - std::vector *oshape) { + mxnet::ShapeVector *ishape, + mxnet::ShapeVector *oshape) { const auto& param = nnvm::get(attrs.parsed); if (ishape->size() != 1 || (*ishape)[0].ndim() != 3) return false; @@ -319,8 +318,8 @@ struct MakeBorderParam : public dmlc::Parameter { DMLC_REGISTER_PARAMETER(MakeBorderParam); inline bool MakeBorderShape(const nnvm::NodeAttrs& attrs, - std::vector *ishape, - std::vector *oshape) { + mxnet::ShapeVector *ishape, + mxnet::ShapeVector *oshape) { const auto& param = nnvm::get(attrs.parsed); if (ishape->size() != 1 || (*ishape)[0].ndim() != 3) return false; @@ -382,7 +381,7 @@ NNVM_REGISTER_OP(_cvimresize) .set_num_inputs(1) .set_num_outputs(1) .set_attr_parser(op::ParamParser) -.set_attr("FInferShape", ResizeShape) +.set_attr("FInferShape", ResizeShape) .set_attr("FInferType", op::ElemwiseType<1, 1>) .set_attr("FCompute", Imresize) .add_argument("src", "NDArray", "source image") @@ -393,7 +392,7 @@ NNVM_REGISTER_OP(_cvcopyMakeBorder) .set_num_inputs(1) .set_num_outputs(1) .set_attr_parser(op::ParamParser) -.set_attr("FInferShape", MakeBorderShape) +.set_attr("FInferShape", MakeBorderShape) .set_attr("FInferType", op::ElemwiseType<1, 1>) .set_attr("FCompute", copyMakeBorder) .add_argument("src", "NDArray", "source image") diff --git a/src/io/image_iter_common.h b/src/io/image_iter_common.h index 10cd8ab4e5de..4bbcb9d21f9a 100644 --- a/src/io/image_iter_common.h +++ b/src/io/image_iter_common.h @@ -118,7 +118,7 @@ struct ImageRecParserParam : public dmlc::Parameter { /*! \brief label-width */ int label_width; /*! \brief input shape */ - TShape data_shape; + mxnet::TShape data_shape; /*! \brief number of threads */ int preprocess_threads; /*! \brief whether to remain silent */ diff --git a/src/io/inst_vector.h b/src/io/inst_vector.h index f06a4e4aabe9..91106e788cdd 100644 --- a/src/io/inst_vector.h +++ b/src/io/inst_vector.h @@ -187,7 +187,7 @@ class TBlobContainer : public TBlob { release(); } } - void resize(const TShape &shape, int type_flag) { + void resize(const mxnet::TShape &shape, int type_flag) { if (tensor_container_) { CHECK_EQ(this->type_flag_, type_flag); this->shape_ = shape; diff --git a/src/io/iter_batchloader.h b/src/io/iter_batchloader.h index be911f695c8e..69eb05f7d729 100644 --- a/src/io/iter_batchloader.h +++ b/src/io/iter_batchloader.h @@ -146,7 +146,7 @@ class BatchLoader : public IIterator { /*! \brief base iterator */ IIterator *base_; /*! \brief data shape */ - std::vector shape_; + mxnet::ShapeVector shape_; /*! \brief unit size */ std::vector unit_size_; // initialize the data holder by using from the first batch. @@ -155,7 +155,7 @@ class BatchLoader : public IIterator { data_.resize(first_batch.data.size()); unit_size_.resize(first_batch.data.size()); for (size_t i = 0; i < first_batch.data.size(); ++i) { - TShape src_shape = first_batch.data[i].shape_; + mxnet::TShape src_shape = first_batch.data[i].shape_; int src_type_flag = first_batch.data[i].type_flag_; // init object attributes std::vector shape_vec; @@ -163,7 +163,7 @@ class BatchLoader : public IIterator { for (index_t dim = 0; dim < src_shape.ndim(); ++dim) { shape_vec.push_back(src_shape[dim]); } - TShape dst_shape(shape_vec.begin(), shape_vec.end()); + mxnet::TShape dst_shape(shape_vec.begin(), shape_vec.end()); shape_[i] = dst_shape; data_[i].resize(mshadow::Shape1(dst_shape.Size()), src_type_flag); unit_size_[i] = src_shape.Size(); diff --git a/src/io/iter_csv.cc b/src/io/iter_csv.cc index 5fd149535be2..0c1b82355410 100644 --- a/src/io/iter_csv.cc +++ b/src/io/iter_csv.cc @@ -37,11 +37,11 @@ struct CSVIterParam : public dmlc::Parameter { /*! \brief path to data csv file */ std::string data_csv; /*! \brief data shape */ - TShape data_shape; + mxnet::TShape data_shape; /*! \brief path to label csv file */ std::string label_csv; /*! \brief label shape */ - TShape label_shape; + mxnet::TShape label_shape; // declare parameters DMLC_DECLARE_PARAMETER(CSVIterParam) { DMLC_DECLARE_FIELD(data_csv) @@ -52,7 +52,7 @@ struct CSVIterParam : public dmlc::Parameter { .describe("The input CSV file or a directory path. " "If NULL, all labels will be returned as 0."); index_t shape1[] = {1}; - DMLC_DECLARE_FIELD(label_shape).set_default(TShape(shape1, shape1 + 1)) + DMLC_DECLARE_FIELD(label_shape).set_default(mxnet::TShape(shape1, shape1 + 1)) .describe("The shape of one label."); } }; @@ -148,7 +148,7 @@ class CSVIterTyped: public CSVIterBase { } private: - inline TBlob AsTBlob(const dmlc::Row& row, const TShape& shape) { + inline TBlob AsTBlob(const dmlc::Row& row, const mxnet::TShape& shape) { CHECK_EQ(row.length, shape.Size()) << "The data size in CSV do not match size of shape: " << "specified shape=" << shape << ", the csv row-length=" << row.length; diff --git a/src/io/iter_image_det_recordio.cc b/src/io/iter_image_det_recordio.cc index 8bfded75f098..876c07520f52 100644 --- a/src/io/iter_image_det_recordio.cc +++ b/src/io/iter_image_det_recordio.cc @@ -151,7 +151,7 @@ struct ImageDetRecParserParam : public dmlc::Parameter { /*! \brief label-width, use -1 for variable width */ int label_width; /*! \brief input shape */ - TShape data_shape; + mxnet::TShape data_shape; /*! \brief number of threads */ int preprocess_threads; /*! \brief whether to remain silent */ diff --git a/src/io/iter_image_recordio_2.cc b/src/io/iter_image_recordio_2.cc index 00c38198659f..0834dd7786ee 100644 --- a/src/io/iter_image_recordio_2.cc +++ b/src/io/iter_image_recordio_2.cc @@ -278,12 +278,12 @@ inline bool ImageRecordIOParser2::ParseNext(DataBatch *out) { for (index_t dim = 0; dim < param_.data_shape.ndim(); ++dim) { shape_vec.push_back(param_.data_shape[dim]); } - TShape data_shape(shape_vec.begin(), shape_vec.end()); + mxnet::TShape data_shape(shape_vec.begin(), shape_vec.end()); shape_vec.clear(); shape_vec.push_back(batch_param_.batch_size); shape_vec.push_back(param_.label_width); - TShape label_shape(shape_vec.begin(), shape_vec.end()); + mxnet::TShape label_shape(shape_vec.begin(), shape_vec.end()); auto ctx = Context::CPU(0); auto dev_id = param_.device_id; @@ -372,6 +372,7 @@ void ImageRecordIOParser2::ProcessImage(const cv::Mat& res, float RGBA_MULT[4] = { 0 }; float RGBA_BIAS[4] = { 0 }; float RGBA_MEAN[4] = { 0 }; + int16_t RGBA_MEAN_INT[4] = {0}; mshadow::Tensor& data = (*data_ptr); if (!std::is_same::value) { RGBA_MULT[0] = contrast_scaled / normalize_param_.std_r; @@ -387,6 +388,10 @@ void ImageRecordIOParser2::ProcessImage(const cv::Mat& res, RGBA_MEAN[1] = normalize_param_.mean_g; RGBA_MEAN[2] = normalize_param_.mean_b; RGBA_MEAN[3] = normalize_param_.mean_a; + RGBA_MEAN_INT[0] = std::round(normalize_param_.mean_r); + RGBA_MEAN_INT[1] = std::round(normalize_param_.mean_g); + RGBA_MEAN_INT[2] = std::round(normalize_param_.mean_b); + RGBA_MEAN_INT[3] = std::round(normalize_param_.mean_a); } } @@ -408,17 +413,30 @@ void ImageRecordIOParser2::ProcessImage(const cv::Mat& res, for (int i = 0; i < res.rows; ++i) { const uchar* im_data = res.ptr(i); for (int j = 0; j < res.cols; ++j) { - for (int k = 0; k < n_channels; ++k) { - RGBA[k] = im_data[swap_indices[k]]; - } - if (!std::is_same::value) { - // normalize/mirror here to avoid memory copies - // logic from iter_normalize.h, function SetOutImg + if (std::is_same::value) { + if (meanfile_ready_) { + for (int k = 0; k < n_channels; ++k) { + RGBA[k] = cv::saturate_cast(im_data[swap_indices[k]] - + static_cast(std::round(meanimg_[k][i][j]))); + } + } else { + for (int k = 0; k < n_channels; ++k) { + RGBA[k] = cv::saturate_cast(im_data[swap_indices[k]] - RGBA_MEAN_INT[k]); + } + } + } else { for (int k = 0; k < n_channels; ++k) { - if (meanfile_ready_) { - RGBA[k] = (RGBA[k] - meanimg_[k][i][j]) * RGBA_MULT[k] + RGBA_BIAS[k]; - } else { - RGBA[k] = (RGBA[k] - RGBA_MEAN[k]) * RGBA_MULT[k] + RGBA_BIAS[k]; + RGBA[k] = im_data[swap_indices[k]]; + } + if (!std::is_same::value) { + // normalize/mirror here to avoid memory copies + // logic from iter_normalize.h, function SetOutImg + for (int k = 0; k < n_channels; ++k) { + if (meanfile_ready_) { + RGBA[k] = (RGBA[k] - meanimg_[k][i][j]) * RGBA_MULT[k] + RGBA_BIAS[k]; + } else { + RGBA[k] = (RGBA[k] - RGBA_MEAN[k]) * RGBA_MULT[k] + RGBA_BIAS[k]; + } } } } @@ -795,5 +813,22 @@ the data type instead of ``float``. .set_body([]() { return new ImageRecordIter2(); }); + +MXNET_REGISTER_IO_ITER(ImageRecordInt8Iter) +.describe(R"code(Iterating on image RecordIO files + +This iterator is identical to ``ImageRecordIter`` except for using ``int8`` as +the data type instead of ``float``. + +)code" ADD_FILELINE) +.add_arguments(ImageRecParserParam::__FIELDS__()) +.add_arguments(ImageRecordParam::__FIELDS__()) +.add_arguments(BatchParam::__FIELDS__()) +.add_arguments(PrefetcherParam::__FIELDS__()) +.add_arguments(ListDefaultAugParams()) +.set_body([]() { + return new ImageRecordIter2(); + }); + } // namespace io } // namespace mxnet diff --git a/src/io/iter_libsvm.cc b/src/io/iter_libsvm.cc index 8abb768ad4c8..3decc7b33e04 100644 --- a/src/io/iter_libsvm.cc +++ b/src/io/iter_libsvm.cc @@ -36,11 +36,11 @@ struct LibSVMIterParam : public dmlc::Parameter { /*! \brief path to data libsvm file */ std::string data_libsvm; /*! \brief data shape */ - TShape data_shape; + mxnet::TShape data_shape; /*! \brief path to label libsvm file */ std::string label_libsvm; /*! \brief label shape */ - TShape label_shape; + mxnet::TShape label_shape; /*! \brief partition the data into multiple parts */ int num_parts; /*! \brief the index of the part will read*/ @@ -55,7 +55,7 @@ struct LibSVMIterParam : public dmlc::Parameter { .describe("The input LibSVM label file or a directory path. " "If NULL, all labels will be read from ``data_libsvm``."); index_t shape1[] = {1}; - DMLC_DECLARE_FIELD(label_shape).set_default(TShape(shape1, shape1 + 1)) + DMLC_DECLARE_FIELD(label_shape).set_default(mxnet::TShape(shape1, shape1 + 1)) .describe("The shape of one label."); DMLC_DECLARE_FIELD(num_parts).set_default(1) .describe("partition the data into multiple parts"); @@ -153,7 +153,7 @@ class LibSVMIter: public SparseIIterator { return param_.label_shape.Size() > 1 ? kCSRStorage : kDefaultStorage; } - virtual const TShape GetShape(bool is_data) const { + virtual const mxnet::TShape GetShape(bool is_data) const { if (is_data) return param_.data_shape; return param_.label_shape; } @@ -161,13 +161,13 @@ class LibSVMIter: public SparseIIterator { private: inline TBlob AsDataBlob(const dmlc::Row& row) { const real_t* ptr = row.value; - TShape shape(mshadow::Shape1(row.length)); + mxnet::TShape shape(mshadow::Shape1(row.length)); return TBlob((real_t*) ptr, shape, cpu::kDevMask); // NOLINT(*) } inline TBlob AsIdxBlob(const dmlc::Row& row) { const uint64_t* ptr = row.index; - TShape shape(mshadow::Shape1(row.length)); + mxnet::TShape shape(mshadow::Shape1(row.length)); return TBlob((int64_t*) ptr, shape, cpu::kDevMask, mshadow::kInt64); // NOLINT(*) } diff --git a/src/io/iter_mnist.cc b/src/io/iter_mnist.cc index 139cf47d63ed..0163a62400f7 100644 --- a/src/io/iter_mnist.cc +++ b/src/io/iter_mnist.cc @@ -104,7 +104,7 @@ class MNISTIter: public IIterator { out_.batch_size = param_.batch_size; if (param_.shuffle) this->Shuffle(); if (param_.silent == 0) { - TShape s; + mxnet::TShape s; s = batch_data_.shape_; if (param_.flat) { LOG(INFO) << "MNISTIter: load " << (unsigned)img_.size(0) << " images, shuffle=" diff --git a/src/io/iter_sparse.h b/src/io/iter_sparse.h index beaf5c682998..22b1836be419 100644 --- a/src/io/iter_sparse.h +++ b/src/io/iter_sparse.h @@ -38,7 +38,7 @@ class SparseIIterator : public IIterator { /*! \brief storage type of the data or label */ virtual const NDArrayStorageType GetStorageType(bool is_data) const = 0; /*! \brief shape of the data or label */ - virtual const TShape GetShape(bool is_data) const = 0; + virtual const mxnet::TShape GetShape(bool is_data) const = 0; }; // class SparseIIterator } // namespace mxnet diff --git a/src/io/iter_sparse_batchloader.h b/src/io/iter_sparse_batchloader.h index 398d6e00fe7b..17c509a0f56b 100644 --- a/src/io/iter_sparse_batchloader.h +++ b/src/io/iter_sparse_batchloader.h @@ -108,14 +108,14 @@ class SparseBatchLoader : public BatchLoader, public SparseIIterator return sparse_base_->GetStorageType(is_data); } - virtual const TShape GetShape(bool is_data) const { - TShape inst_shape = sparse_base_->GetShape(is_data); + virtual const mxnet::TShape GetShape(bool is_data) const { + mxnet::TShape inst_shape = sparse_base_->GetShape(is_data); std::vector shape_vec; shape_vec.push_back(param_.batch_size); for (index_t dim = 0; dim < inst_shape.ndim(); ++dim) { shape_vec.push_back(inst_shape[dim]); } - return TShape(shape_vec.begin(), shape_vec.end()); + return mxnet::TShape(shape_vec.begin(), shape_vec.end()); } private: @@ -186,7 +186,7 @@ class SparseBatchLoader : public BatchLoader, public SparseIIterator // allocate buffer for (size_t i = 0; i < num_arrays; ++i) { // init object attributes - TShape dst_shape(mshadow::Shape1(buff_sizes[i])); + mxnet::TShape dst_shape(mshadow::Shape1(buff_sizes[i])); data_[i].resize(mshadow::Shape1(buff_sizes[i]), dtypes_[i]); CHECK(data_[i].dptr_ != nullptr); } diff --git a/src/io/iter_sparse_prefetcher.h b/src/io/iter_sparse_prefetcher.h index 3908f9bd3826..3f06052b0292 100644 --- a/src/io/iter_sparse_prefetcher.h +++ b/src/io/iter_sparse_prefetcher.h @@ -134,7 +134,7 @@ class SparsePrefetcherIter : public PrefetcherIter { return sparse_loader_->GetStorageType(is_data); } - virtual const TShape GetShape(bool is_data) const { + virtual const mxnet::TShape GetShape(bool is_data) const { return sparse_loader_->GetShape(is_data); } diff --git a/src/kvstore/comm.h b/src/kvstore/comm.h index 08f6155cb5b4..88e363b5de7d 100644 --- a/src/kvstore/comm.h +++ b/src/kvstore/comm.h @@ -50,7 +50,7 @@ class Comm { * \brief init key with the data shape and storage shape */ virtual void Init(int key, const NDArrayStorageType stype, - const TShape& shape, int dtype = mshadow::kFloat32) = 0; + const mxnet::TShape& shape, int dtype = mshadow::kFloat32) = 0; /** * \brief returns src[0] + .. + src[src.size()-1] */ @@ -110,7 +110,7 @@ class CommCPU : public Comm { } virtual ~CommCPU() { } - void Init(int key, const NDArrayStorageType stype, const TShape& shape, + void Init(int key, const NDArrayStorageType stype, const mxnet::TShape& shape, int type = mshadow::kFloat32) override { // Delayed allocation - the dense merged buffer might not be used at all if push() // only sees sparse arrays @@ -456,7 +456,7 @@ class CommDevice : public Comm { virtual ~CommDevice() { } - void Init(int key, const NDArrayStorageType stype, const TShape& shape, + void Init(int key, const NDArrayStorageType stype, const mxnet::TShape& shape, int dtype = mshadow::kFloat32) override { sorted_key_attrs_.emplace_back(key, shape, dtype); inited_ = false; @@ -568,9 +568,9 @@ class CommDevice : public Comm { false, buf.merged.dtype()); buf.residual[i] = 0; int64_t small_size = gc_->GetCompressedSize(buf.merged.shape().Size()); - buf.compressed_recv_buf[i] = NDArray(TShape{small_size}, buf.merged.ctx(), + buf.compressed_recv_buf[i] = NDArray(mxnet::TShape{small_size}, buf.merged.ctx(), false, buf.merged.dtype()); - buf.compressed_send_buf[i] = NDArray(TShape{small_size}, src[i].ctx(), + buf.compressed_send_buf[i] = NDArray(mxnet::TShape{small_size}, src[i].ctx(), false, buf.merged.dtype()); } } @@ -673,7 +673,7 @@ class CommDevice : public Comm { } } - using KeyAttrs = std::tuple; + using KeyAttrs = std::tuple; // try to allocate buff on device evenly void InitMergeBuffer(const std::vector& devs) { std::sort(sorted_key_attrs_.begin(), sorted_key_attrs_.end(), []( @@ -688,7 +688,7 @@ class CommDevice : public Comm { for (auto& sorted_key_attr : sorted_key_attrs_) { const int key = std::get<0>(sorted_key_attr); - const TShape& shape = std::get<1>(sorted_key_attr); + const mxnet::TShape& shape = std::get<1>(sorted_key_attr); const int type = std::get<2>(sorted_key_attr); auto& buf = merge_buf_[key]; Context ctx; diff --git a/src/kvstore/comm_tree.h b/src/kvstore/comm_tree.h index 11d99c021917..11ca2d6528be 100644 --- a/src/kvstore/comm_tree.h +++ b/src/kvstore/comm_tree.h @@ -58,7 +58,7 @@ class CommDeviceTree : public CommDevice { virtual ~CommDeviceTree() { } - void Init(int key, const NDArrayStorageType stype, const TShape& shape, + void Init(int key, const NDArrayStorageType stype, const mxnet::TShape& shape, int dtype = mshadow::kFloat32) override { tree_sorted_key_attrs_.emplace_back(key, shape, dtype); sorted_key_attrs_.emplace_back(key, shape, dtype); @@ -385,7 +385,7 @@ class CommDeviceTree : public CommDevice { #endif } - using KeyAttrs = std::tuple; + using KeyAttrs = std::tuple; // try to allocate buff on device evenly void InitMergeBufferTree() { LOG(INFO) << "Using Tree"; @@ -402,7 +402,7 @@ class CommDeviceTree : public CommDevice { for (auto& tree_sorted_key_attr : tree_sorted_key_attrs_) { const int key = std::get<0>(tree_sorted_key_attr); - const TShape& shape = std::get<1>(tree_sorted_key_attr); + const mxnet::TShape& shape = std::get<1>(tree_sorted_key_attr); const int type = std::get<2>(tree_sorted_key_attr); if (key_dist.find(shape.Size()) == key_dist.end()) @@ -444,7 +444,7 @@ class CommDeviceTree : public CommDevice { // buf.merged enforces that we only visit each GPU once if (buf.merged.empty()) { - TShape shape_copy = shape; + mxnet::TShape shape_copy = shape; int total_size = shape.Size(); unsigned first_size = shape[0]; if (total_size > gpuarray_bound_ && first_size >= 2*devs_.size()) { diff --git a/src/kvstore/kvstore_dist.h b/src/kvstore/kvstore_dist.h index 23fbf67474ee..9fe41c51a2c6 100644 --- a/src/kvstore/kvstore_dist.h +++ b/src/kvstore/kvstore_dist.h @@ -383,8 +383,9 @@ class KVStoreDist : public KVStoreLocal { // Init the small buffer and residual_ buffer for quantize if (small_buf.is_none()) { - small_buf = NDArray(TShape{pskv.size}, comm_buf.ctx(), false, dtype); - res_buf = NDArray(TShape{static_cast(original_size)}, comm_buf.ctx(), false, dtype); + small_buf = NDArray(mxnet::TShape{pskv.size}, comm_buf.ctx(), false, dtype); + res_buf = NDArray(mxnet::TShape{static_cast(original_size)}, + comm_buf.ctx(), false, dtype); res_buf = 0; } gradient_compression_->Quantize(comm_buf, &small_buf, &res_buf, priority); diff --git a/src/kvstore/kvstore_dist_server.h b/src/kvstore/kvstore_dist_server.h index 372b58dbbf3d..0cb1a11e3fcc 100644 --- a/src/kvstore/kvstore_dist_server.h +++ b/src/kvstore/kvstore_dist_server.h @@ -459,7 +459,7 @@ class KVStoreDistServer { auto unit_len = req_data.lens[1] / num_bytes; CHECK_GT(unit_len, 0); size_t ds[] = {num_rows, (size_t) unit_len}; - TShape dshape(ds, ds + 2); + mxnet::TShape dshape(ds, ds + 2); CHECK_EQ(req_data.vals.size(), num_rows * unit_len * num_bytes); TBlob recv_blob; MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { @@ -546,7 +546,7 @@ class KVStoreDistServer { // data TBlob idx_blob(indices.data(), mshadow::Shape1(num_rows), cpu::kDevMask); size_t ds[] = {(size_t) num_rows, (size_t) unit_len}; - TShape dshape(ds, ds + 2); + mxnet::TShape dshape(ds, ds + 2); TBlob recv_blob; MSHADOW_REAL_TYPE_SWITCH(type.dtype, DType, { recv_blob = TBlob(reinterpret_cast(req_data.vals.data()), @@ -620,12 +620,12 @@ class KVStoreDistServer { auto& stored = store_[key]; size_t ds[] = {(size_t)req_data.lens[1] / mshadow::mshadow_sizeof(type.dtype)}; - TShape dshape(ds, ds + 1); + mxnet::TShape dshape(ds, ds + 1); TBlob recv_blob(reinterpret_cast(req_data.vals.data()), dshape, cpu::kDevMask); NDArray recved = NDArray(recv_blob, 0); NDArray decomp_buf = decomp_buf_[key]; - dshape = TShape{(int64_t) original_size}; + dshape = mxnet::TShape{(int64_t) original_size}; if (decomp_buf.is_none()) { decomp_buf = NDArray(dshape, Context()); @@ -684,7 +684,7 @@ class KVStoreDistServer { // the operators with \a NDArray are actually finished if (req_meta.push) { size_t ds[] = {(size_t) req_data.lens[0] / mshadow::mshadow_sizeof(type.dtype)}; - TShape dshape(ds, ds + 1); + mxnet::TShape dshape(ds, ds + 1); TBlob recv_blob; MSHADOW_REAL_TYPE_SWITCH(type.dtype, DType, { recv_blob = TBlob(reinterpret_cast(req_data.vals.data()), dshape, cpu::kDevMask); diff --git a/src/kvstore/kvstore_nccl.h b/src/kvstore/kvstore_nccl.h index a4ba533917b8..0c1411002e7f 100644 --- a/src/kvstore/kvstore_nccl.h +++ b/src/kvstore/kvstore_nccl.h @@ -443,7 +443,7 @@ class KVStoreNCCL : public KVStoreLocal { } // Initialize single key - void InitKey(int key, const NDArrayStorageType stype, const TShape& shape, + void InitKey(int key, const NDArrayStorageType stype, const mxnet::TShape& shape, int dtype = mshadow::kFloat32) { if (stype == kDefaultStorage) { key_attrs_.push_back(std::make_tuple(key, shape, dtype)); @@ -492,11 +492,11 @@ class KVStoreNCCL : public KVStoreLocal { } } - using KeyAttrs = std::tuple; + using KeyAttrs = std::tuple; void InitMergeBuffer(const std::vector& devs) { for (size_t i = 0; i < key_attrs_.size(); ++i) { int key = std::get<0>(key_attrs_[i]); - TShape s = std::get<1>(key_attrs_[i]); + mxnet::TShape s = std::get<1>(key_attrs_[i]); int type = std::get<2>(key_attrs_[i]); auto& buf = merge_buf_[key]; // always use devs[0] as root diff --git a/src/mxfeatures.cc b/src/libinfo.cc similarity index 76% rename from src/mxfeatures.cc rename to src/libinfo.cc index 7a435d7c81c9..2af61eac9eca 100644 --- a/src/mxfeatures.cc +++ b/src/libinfo.cc @@ -19,12 +19,14 @@ /*! * Copyright (c) 2018 by Contributors - * \file mxfeatures.cc + * \file libinfo.cc + * \author larroy * \brief check MXNet features including compile time support */ -#include "mxnet/mxfeatures.h" +#include "mxnet/libinfo.h" #include +#include "mxnet/base.h" namespace mxnet { namespace features { @@ -108,5 +110,53 @@ bool is_enabled(const unsigned feat) { return featureSet.is_enabled(feat); } +LibInfo::LibInfo() { + for (size_t i = 0; i < MAX_FEATURES; ++i) { + m_lib_features[i].name = EnumNames::names[i].c_str(); + m_lib_features[i].enabled = is_enabled(i); + } +} + +LibInfo *LibInfo::getInstance() { + if (!m_inst) + m_inst = std::make_unique(); + return m_inst.get(); +} + +std::unique_ptr LibInfo::m_inst = nullptr; + +const std::vector EnumNames::names = { + "CUDA", + "CUDNN", + "NCCL", + "CUDA_RTC", + "TENSORRT", + "CPU_SSE", + "CPU_SSE2", + "CPU_SSE3", + "CPU_SSE4_1", + "CPU_SSE4_2", + "CPU_SSE4A", + "CPU_AVX", + "CPU_AVX2", + "OPENMP", + "SSE", + "F16C", + "JEMALLOC", + "BLAS_OPEN", + "BLAS_ATLAS", + "BLAS_MKL", + "BLAS_APPLE", + "LAPACK", + "MKLDNN", + "OPENCV", + "CAFFE", + "PROFILER", + "DIST_KVSTORE", + "CXX14", + "SIGNAL_HANDLER", + "DEBUG", +}; + } // namespace features } // namespace mxnet diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 0f0fed24d4e6..367712755483 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -50,9 +50,9 @@ DMLC_REGISTRY_ENABLE(::mxnet::NDArrayFunctionReg); namespace mxnet { -NDArray::NDArray(const NDArrayStorageType stype, const TShape &shape, Context ctx, +NDArray::NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, Context ctx, bool delay_alloc, int dtype, std::vector aux_types, - std::vector aux_shapes, TShape storage_shape) : shape_(shape), + mxnet::ShapeVector aux_shapes, mxnet::TShape storage_shape) : shape_(shape), dtype_(dtype), storage_type_(stype), entry_({nullptr, 0, 0}) { // Assign default aux types if not given if (aux_types.size() == 0 @@ -70,10 +70,10 @@ NDArray::NDArray(const NDArrayStorageType stype, const TShape &shape, Context ct if (aux_shapes.size() == 0 && stype != kDefaultStorage) { if (stype == kRowSparseStorage) { - aux_shapes = {TShape(mshadow::Shape1(0))}; + aux_shapes = {mxnet::TShape(mshadow::Shape1(0))}; } else if (stype == kCSRStorage) { // aux shapes for indptr and indices - aux_shapes = {TShape(mshadow::Shape1(0)), TShape(mshadow::Shape1(0))}; + aux_shapes = {mxnet::TShape(mshadow::Shape1(0)), mxnet::TShape(mshadow::Shape1(0))}; } else { LOG(FATAL) << "Unknown storage type " << stype; } @@ -129,7 +129,7 @@ NDArray::Chunk::~Chunk() { }, shandle.ctx, var); } -void NDArray::Chunk::CheckAndAllocData(const TShape &shape, int dtype) { +void NDArray::Chunk::CheckAndAllocData(const mxnet::TShape &shape, int dtype) { CHECK_NE(aux_shapes.size(), 0) << "data is expected to be allocated after aux_data"; auto dbytes = shape.Size() * mshadow::mshadow_sizeof(dtype); @@ -171,7 +171,7 @@ nnvm::Symbol NDArray::get_autograd_symbol() const { NDArray::NDArray(mkldnn::memory::primitive_desc mem_pd) : storage_type_(kDefaultStorage), entry_({nullptr, 0, 0}) { auto mem_desc = mem_pd.desc(); - shape_ = TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); + shape_ = mxnet::TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); dtype_ = get_mxnet_type(mem_desc.data.data_type); ptr_ = std::make_shared(shape_, Context::CPU(), true, dtype_); ptr_->CheckAndAlloc(mem_pd.get_size()); @@ -182,7 +182,7 @@ NDArray::NDArray(const std::shared_ptr &mkldnn_mem) : storage_type_(kDefaultStorage), entry_({nullptr, 0, 0}) { auto mem_pd = mkldnn_mem->get_primitive_desc(); auto mem_desc = mem_pd.desc(); - shape_ = TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); + shape_ = mxnet::TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); dtype_ = get_mxnet_type(mem_desc.data.data_type); ptr_ = std::make_shared(shape_, Context::CPU(), true, dtype_); ptr_->shandle.dptr = mkldnn_mem->get_data_handle(); @@ -192,7 +192,7 @@ NDArray::NDArray(const std::shared_ptr &mkldnn_mem) ptr_->static_data = true; } -NDArray NDArray::MKLDNNDataReshape(const TShape &shape) const { +NDArray NDArray::MKLDNNDataReshape(const mxnet::TShape &shape) const { CHECK(!is_none()) << "NDArray is not initialized"; CHECK_GE(shape_.Size(), shape.Size()) << "NDArray.Reshape: target shape size is larger current shape"; @@ -231,7 +231,7 @@ NDArray NDArray::MKLDNNDataReshape(const TShape &shape) const { #endif -NDArray NDArray::Reshape(const TShape &shape) const { +NDArray NDArray::Reshape(const mxnet::TShape &shape) const { CHECK(!is_none()) << "NDArray is not initialized"; CHECK_GE(shape_.Size(), shape.Size()) << "NDArray.Reshape: target shape size is larger current shape"; @@ -246,7 +246,7 @@ NDArray NDArray::Reshape(const TShape &shape) const { return ret; } -NDArray NDArray::ReshapeWithRecord(const TShape &shape) { +NDArray NDArray::ReshapeWithRecord(const mxnet::TShape &shape) { NDArray ret = this->Reshape(shape); if (!Imperative::Get()->is_recording()) return ret; @@ -300,7 +300,7 @@ NDArray NDArray::At(index_t idx) const { << "Storage type " << storage_type() << " doesn't support At()"; NDArray ret = this->Slice(idx, idx+1); if (shape_.ndim() > 1) { - return ret.Reshape(TShape(shape_.data()+1, shape_.data()+shape_.ndim())); + return ret.Reshape(mxnet::TShape(shape_.data()+1, shape_.data()+shape_.ndim())); } else { return ret; } @@ -311,7 +311,7 @@ NDArray NDArray::AtWithRecord(index_t idx) { << "Storage type " << storage_type() << " doesn't support At()"; NDArray ret = this->SliceWithRecord(idx, idx+1); if (shape_.ndim() > 1) { - return ret.ReshapeWithRecord(TShape(shape_.data()+1, shape_.data()+shape_.ndim())); + return ret.ReshapeWithRecord(mxnet::TShape(shape_.data()+1, shape_.data()+shape_.ndim())); } else { return ret; } @@ -325,13 +325,13 @@ NDArray NDArray::aux_ndarray(size_t i) const { CHECK_NE(storage_type(), kDefaultStorage); CHECK(i < ptr_->aux_shapes.size()); // create a delay_alloc default ndarray as output - NDArray ret(TShape(), ctx(), true, aux_type(i)); + NDArray ret(mxnet::TShape(), ctx(), true, aux_type(i)); ret.SyncCopyFromNDArray(*this, i); return ret; } NDArray NDArray::data_ndarray() const { - NDArray ret(TShape(), ctx(), true, dtype_); + NDArray ret(mxnet::TShape(), ctx(), true, dtype_); ret.SyncCopyFromNDArray(*this); return ret; } @@ -454,7 +454,7 @@ void NDArray::Chunk::MKLDNNDataReorder(const mkldnn::memory::primitive_desc &pd) mkl_mem_.reset(new MKLDNNMemory(pd, shandle.dptr)); } -void NDArray::Chunk::SetMKLMem(const TShape &shape, int dtype) { +void NDArray::Chunk::SetMKLMem(const mxnet::TShape &shape, int dtype) { // The shape of the array and the one of the MKL memory may mismatch. // For example, if the array stores parameters, the MKL memory may store data // in 5 dimensions while the NDArray stores data in 4 dimensions. @@ -549,7 +549,7 @@ const mkldnn::memory *NDArray::GetMKLDNNDataReorder( // If they have different shapes, we need to reshape the array first. // Since this method will only be used inside an operator, we can call // MKLDNNDataReshape to reshape an array. - TShape required_shape(desc2.data.ndims); + mxnet::TShape required_shape(desc2.data.ndims); for (int i = 0; i < desc2.data.ndims; i++) required_shape[i] = desc2.data.dims[i]; NDArray reshaped = MKLDNNDataReshape(required_shape); @@ -575,7 +575,7 @@ NDArray NDArray::Reorder2Default() const { // create new ndarray from mkldnn layout mkldnn::memory::desc from_desc = ptr_->mkl_mem_->GetPrimitiveDesc().desc(); - TShape tshape(from_desc.data.ndims); + mxnet::TShape tshape(from_desc.data.ndims); for (int i = 0; i < from_desc.data.ndims; i++) tshape[i] = from_desc.data.dims[i]; NDArray ret(tshape, ctx(), false, dtype()); mkldnn::memory::primitive_desc def_pd = ptr_->mkl_mem_->GetPrimitiveDesc(format); @@ -737,7 +737,7 @@ void NDArray::UpdateMKLDNNMemDesc(mkldnn::memory::format format) { void NDArray::SetTBlob() const { CHECK(ptr_ != nullptr); - TShape shape = shape_; + mxnet::TShape shape = shape_; char *dptr = static_cast(ptr_->shandle.dptr); auto stype = storage_type(); if (stype == kDefaultStorage) { @@ -1161,7 +1161,7 @@ void CopyFromToImpl(const NDArray& from, const NDArray& to, if (from_stype == to_stype) { casted_nd = from; // same stype, no need to cast from } else { // different stypes on different ctx needs an temporary casted_nd - const TShape& shape = from.shape(); + const mxnet::TShape& shape = from.shape(); if (to_stype == kDefaultStorage) { casted_nd = NDArray(shape, from_ctx); } else { @@ -1567,7 +1567,7 @@ NDArray &NDArray::operator/=(const real_t &src) { return ScalarOpApply(this, src); } -/* magic number for ndarray version 1, with int64_t TShape */ +/* magic number for ndarray version 1, with int64_t mxnet::TShape */ static const uint32_t NDARRAY_V1_MAGIC = 0xF993fac8; /* magic number for ndarray version 2, with storage type */ @@ -1643,14 +1643,14 @@ void NDArray::Save(dmlc::Stream *strm) const { } } -bool LegacyTShapeLoad(dmlc::Stream *strm, TShape *shape, const uint32_t magic) { +bool LegacyTShapeLoad(dmlc::Stream *strm, mxnet::TShape *shape, const uint32_t magic) { switch (magic) { case NDARRAY_V1_MAGIC: return shape->Load(strm); default: - // meet legacy TShape, magic is ndim here + // meet legacy mxnet::TShape, magic is ndim here uint32_t ndim = magic; - *shape = TShape(ndim); + *shape = mxnet::TShape(ndim); std::vector buffer(ndim); size_t nread = ndim * sizeof(uint32_t); if (strm->Read(buffer.data(), nread) != nread) return false; @@ -1661,7 +1661,7 @@ bool LegacyTShapeLoad(dmlc::Stream *strm, TShape *shape, const uint32_t magic) { bool NDArray::LegacyLoad(dmlc::Stream *strm, const uint32_t magic) { // load shape - TShape shape; + mxnet::TShape shape; if (!LegacyTShapeLoad(strm, &shape, magic)) return false; if (shape.ndim() == 0) { *this = NDArray(); return true; @@ -1703,13 +1703,13 @@ bool NDArray::Load(dmlc::Stream *strm) { const int32_t nad = num_aux_data(static_cast(stype)); // load storage shape - TShape sshape; + mxnet::TShape sshape; if (nad > 0) { if (!sshape.Load(strm)) return false; } // load shape - TShape shape; + mxnet::TShape shape; if (!shape.Load(strm)) return false; if (shape.ndim() == 0) { *this = NDArray(); return true; @@ -1725,7 +1725,7 @@ bool NDArray::Load(dmlc::Stream *strm) { // load aux_types and aux_shapes std::vector aux_types; - std::vector aux_shapes; + mxnet::ShapeVector aux_shapes; if (nad > 0) { aux_types.resize(nad); aux_shapes.resize(nad); @@ -1819,14 +1819,14 @@ NDArray NDArray::Copy(Context ctx) const { } void NDArray::SyncCopyFromCPU(const void *data, size_t size) const { - TShape dshape = this->shape(); + mxnet::TShape dshape = this->shape(); CHECK_EQ(dshape.Size(), size) << "Memory size do not match"; TBlob src((void*)data, dshape, cpu::kDevMask, this->dtype_, 0); // NOLINT(*) if (this->ctx().dev_mask() == cpu::kDevMask) { this->WaitToWrite(); - RunContext rctx{this->ctx(), nullptr}; + RunContext rctx{this->ctx(), nullptr, nullptr, false}; TBlob dst = this->data(); ndarray::Copy(src, &dst, Context::CPU(), Context::CPU(), rctx); } else { @@ -1876,7 +1876,7 @@ void NDArray::SyncCopyFromNDArray(const NDArray& src, int i, int j) { // get or create a dst tblob for copying src to it // if dst is a dense format and has not been allocated, allocate memory for it // else if dst is not initialized, allocate corresponding data blob for it - auto get_dst_data = [&](const TShape& src_shape) { + auto get_dst_data = [&](const mxnet::TShape& src_shape) { if (this->storage_type() == kDefaultStorage) { this->ReshapeAndAlloc(src_shape); } else if (!this->storage_initialized()) { @@ -1950,14 +1950,14 @@ void NDArray::SyncCopyFromNDArray(const NDArray& src, int i, int j) { } void NDArray::SyncCopyToCPU(void *data, size_t size) const { - TShape dshape = this->shape(); + mxnet::TShape dshape = this->shape(); CHECK_EQ(dshape.Size(), size) << "Memory size do not match"; TBlob dst(data, dshape, cpu::kDevMask, this->dtype_, 0); // NOLINT(*) if (this->ctx().dev_mask() == cpu::kDevMask) { this->WaitToRead(); - RunContext rctx{this->ctx(), nullptr}; + RunContext rctx{this->ctx(), nullptr, nullptr, false}; NDArray src = *this; #if MXNET_USE_MKLDNN == 1 if (src.IsMKLDNNData()) @@ -2027,11 +2027,6 @@ MXNET_REGISTER_NDARRAY_FUN(_set_value) MXNET_REGISTER_NDARRAY_FUN(_onehot_encode) .set_function(BinaryOp); -MXNET_REGISTER_NDARRAY_FUN(choose_element_0index) -.set_function(BinaryOp) -.describe("Choose one element from each line(row for python, column for R/Julia)" - " in lhs according to index indicated by rhs." - " This function assume rhs uses 0-based index."); MXNET_REGISTER_NDARRAY_FUN(fill_element_0index) .set_function(TernaryOp) @@ -2056,7 +2051,7 @@ void CopyFromToSimple( NNVM_REGISTER_OP(_copyto) .set_num_inputs(1) .set_num_outputs(1) -.set_attr("FInferShape", op::ElemwiseShape<1, 1>) +.set_attr("FInferShape", op::ElemwiseShape<1, 1>) .set_attr("FInferType", [](const NodeAttrs& attrs, std::vector *in_type, std::vector *out_type) { return !op::type_is_none((*in_type)[0]) && !op::type_is_none((*out_type)[0]); diff --git a/src/ndarray/ndarray_function.h b/src/ndarray/ndarray_function.h index 97c23b67592a..70b626dbb9b7 100644 --- a/src/ndarray/ndarray_function.h +++ b/src/ndarray/ndarray_function.h @@ -38,7 +38,7 @@ namespace mxnet { /*! \brief namespace to support all possible Ndarray operator */ namespace ndarray { struct BinaryBase { - inline static TShape GetShape(const TShape &lshape, const TShape &rshape) { + inline static mxnet::TShape GetShape(const mxnet::TShape &lshape, const mxnet::TShape &rshape) { CHECK(lshape == rshape) << "operands shape mismatch"; CHECK(lshape.ndim() != 0) << "source operand have zero dimension shape"; return lshape; @@ -94,7 +94,7 @@ struct ClipMax : public BinaryBase { struct OneHotEncode { - inline static TShape GetShape(const TShape &index, const TShape &proptype) { + inline static mxnet::TShape GetShape(const mxnet::TShape &index, const mxnet::TShape &proptype) { CHECK(index.ndim() == 1 && proptype.ndim() == 2) << "OneHotEncode only support 1d index."; CHECK_EQ(index[0], proptype[0]) << "OneHotEncode shape inconsistent"; return proptype; @@ -102,7 +102,7 @@ struct OneHotEncode { }; struct MatChooseRowElem { - inline static TShape GetShape(const TShape &lshape, const TShape &rshape) { + inline static mxnet::TShape GetShape(const mxnet::TShape &lshape, const mxnet::TShape &rshape) { CHECK(lshape.ndim() == 2 && rshape.ndim() == 1) << "choose_row_element only support 2D Matrix and 1D index"; CHECK_EQ(lshape[0], rshape[0]) << "choose_row_element index and matrix shape mismatch"; @@ -111,7 +111,9 @@ struct MatChooseRowElem { }; struct MatFillRowElem { - inline static TShape GetShape(const TShape &lshape, const TShape &mshape, const TShape &rshape) { + inline static mxnet::TShape GetShape(const mxnet::TShape &lshape, + const mxnet::TShape &mshape, + const mxnet::TShape &rshape) { CHECK(lshape.ndim() == 2 && mshape.ndim() == 1 && rshape.ndim() == 1) << "fill_row_element only support 2D Matrix, 1D value and 1D index"; CHECK((lshape[0] == mshape[0]) && (mshape[0] == rshape[0])) diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc new file mode 100644 index 000000000000..4927191a5964 --- /dev/null +++ b/src/nnvm/gradient.cc @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2016 by Contributors + * \file gradients.cc + * \brief Passes that takes gradient of the graph + * This code code was modified based on mxnet codebase by Min Lin + */ +#include +#include +#include +#include +#include + +namespace nnvm { +namespace pass { +namespace { + +// default aggregate gradient function +// require operator zeros and elemwise_sum to be presented. +NodeEntry DefaultAggregateGradient(std::vector&& v) { + if (v.size() == 1) { + return std::move(v[0]); + } else if (v.size() == 0) { + NodePtr zero_node = Node::Create(); + zero_node->attrs.op = Op::Get("zeros"); + zero_node->attrs.name = "zero_grad"; + zero_node->attrs.op->attr_parser(&(zero_node->attrs)); + return NodeEntry{zero_node, 0, 0}; + } else { + NodePtr sum_node = Node::Create(); + sum_node->attrs.op = Op::Get("elemwise_sum"); + sum_node->inputs = std::move(v); + sum_node->attrs.name = "grad_sum"; + sum_node->attrs.dict["num_args"] = std::to_string(sum_node->inputs.size()); + sum_node->attrs.op->attr_parser(&(sum_node->attrs)); + return NodeEntry{sum_node, 0, 0}; + } +} + +bool CheckGradAllZero(const std::vector& grads, + const std::vector& zero_ops) { + if (!grads.size() || !zero_ops.size()) return false; + for (const auto& g : grads) { + bool found = false; + for (const auto& op : zero_ops) { + if (g.node->op() == op) { + found = true; + break; + } + } + if (!found) return false; + } + return true; +} + +// helper entry +struct GradEntry { +#ifdef _MSC_VER + NodeEntry sum = NodeEntry{nullptr, 0, 0}; +#else + NodeEntry sum{nullptr, 0, 0}; +#endif + std::vector grads; + bool need_attr_hint{true}; +}; + +Graph Gradient(Graph src) { + using nnvm::FGradient; + using MirrorFun = std::function; + using AttrHintFun = std::function; + + CHECK_NE(src.attrs.count("grad_ys"), 0U) + << "Gradient require grad_ys to be presented."; + CHECK_NE(src.attrs.count("grad_ys_out_grad"), 0U) + << "Gradient require grad_ys_out_grad to be presented."; + CHECK_NE(src.attrs.count("grad_xs"), 0U) + << "Gradient require grad_xs to be presented."; + const std::vector& ys = + src.GetAttr >("grad_ys"); + const std::vector& ys_out_grad = + src.GetAttr >("grad_ys_out_grad"); + const std::vector& xs = + src.GetAttr >("grad_xs"); + using AggFun = std::function&& inputs)>; + AggFun agg_fun = DefaultAggregateGradient; + if (src.attrs.count("grad_aggregate_fun") != 0) { + agg_fun = src.GetAttr("grad_aggregate_fun"); + } + MirrorFun mirror_fun = nullptr; + if (src.attrs.count("grad_mirror_fun") != 0) { + mirror_fun = src.GetAttr("grad_mirror_fun"); + } + AttrHintFun attr_hint_fun = nullptr; + if (src.attrs.count("attr_hint_fun") != 0) { + attr_hint_fun = src.GetAttr("attr_hint_fun"); + } + std::vector zero_ops; + if (src.attrs.count("zero_ops") != 0) { + zero_ops = src.GetAttr >("zero_ops"); + } + const Op* copy_op = (src.attrs.count("copy_op") != 0) ? + Op::Get(src.GetAttr("copy_op")) : + nullptr; + + // topo sort + std::vector topo_order; + std::unordered_map > output_grads; + + DFSVisit(ys, [&](const NodePtr& node) { + if (output_grads.count(node.get()) == 0) { + output_grads[node.get()].resize(node->num_outputs()); + } + topo_order.push_back(node); + }); + + CHECK_EQ(ys.size(), ys_out_grad.size()); + for (size_t i = 0; i < ys.size(); ++i) { + NodeEntry ograd = ys_out_grad[i]; + output_grads[ys[i].node.get()][ys[i].index].grads = { ograd }; + } + + // Check that all xs are reachable from ys + for (size_t i = 0; i < xs.size(); ++i) { + CHECK(output_grads.find(xs[i].node.get()) != output_grads.end()) + << "Cannot differentiate with respect to the " << i+1 << "-th variable " + << "because it is unreachable from the outputs."; + } + + // construct mirror reduece memory strategy if needed + std::unordered_map mirror_map; + if (mirror_fun != nullptr) { + for (const NodePtr& n : topo_order) { + if (mirror_fun(*n)) { + NodePtr new_node = Node::Create(); + *new_node = *n; + new_node->attrs.name += "_mirror"; + for (auto& e : new_node->inputs) { + e.node = mirror_map.at(e.node.get()); + } + for (auto& n : new_node->control_deps) { + n = mirror_map.at(n.get()); + } + mirror_map[n.get()] = std::move(new_node); + } else { + mirror_map[n.get()] = n; + } + } + } + + // traverse backward + static auto& grad_fun_map = Op::GetAttr("FGradient"); + static auto& finfer_shape = Op::GetAttr("FInferShape"); + + std::vector out_agg_grads; + for (auto rit = topo_order.rbegin(); rit != topo_order.rend(); ++rit) { + const NodePtr& ptr = *rit; + if (ptr->is_variable()) continue; + out_agg_grads.clear(); + auto& out_grad_vec = output_grads.at(ptr.get()); + for (uint32_t i = 0; i < out_grad_vec.size(); ++i) { + GradEntry& e = out_grad_vec[i]; + e.sum = agg_fun(std::move(e.grads)); + if (e.need_attr_hint && attr_hint_fun != nullptr) { + e.sum = attr_hint_fun(e.sum, NodeEntry{ptr, 0, i}); + } + out_agg_grads.push_back(e.sum); + } + if ((*rit)->inputs.size() != 0) { + NodePtr fwd_node = (mirror_map.size() == 0 ? ptr : mirror_map.at(ptr.get())); + std::vector input_grads; + if (grad_fun_map.count(ptr->op())) { + input_grads = grad_fun_map[ptr->op()](fwd_node, out_agg_grads); + CHECK_EQ((*rit)->inputs.size(), input_grads.size()) + << "Gradient function not returning enough gradient"; + } else if (CheckGradAllZero(out_agg_grads, zero_ops)) { + for (size_t i = 0; i < fwd_node->num_inputs(); ++i) { + std::ostringstream os; + if (1 == fwd_node->num_inputs()) { + os << fwd_node->attrs.name << "_backward"; + } else { + os << fwd_node->attrs.name << "_in" << i << "_backward"; + } + auto p = Node::Create(); + p->attrs.op = zero_ops[0]; + p->attrs.name = os.str(); + p->inputs.push_back(fwd_node->inputs[i]); + p->control_deps.emplace_back(fwd_node); + if (p->op()->attr_parser != nullptr) { + p->op()->attr_parser(&(p->attrs)); + } + input_grads.emplace_back(nnvm::NodeEntry{p, 0, 0}); + } + } else { + LOG(FATAL) << "Operator " << fwd_node->op()->name << " is non-differentiable " + << "because it didn't register FGradient attribute."; + } + auto git = input_grads.begin(); + for (auto it = (*rit)->inputs.begin(); it != (*rit)->inputs.end(); ++it, ++git) { + auto& ge = output_grads[it->node.get()][it->index]; + // if any of the backward op can do shape inference, the hint is not necessary. + if (finfer_shape.count(git->node->op())) { + ge.need_attr_hint = false; + } + ge.grads.emplace_back(std::move(*git)); + } + } + } + // take out the xs' grads + Graph ret; + ret.outputs.resize(xs.size()); + NodeEntryMap > unique_grads; + size_t counter = 0; + for (const NodeEntry& e : xs) { + GradEntry& entry = output_grads[e.node.get()][e.index]; + // aggregate sum if there haven't been + if (entry.sum.node.get() == nullptr) { + entry.sum = agg_fun(std::move(entry.grads)); + if (entry.need_attr_hint && attr_hint_fun != nullptr) { + entry.sum = attr_hint_fun(entry.sum, e); + } + } + if (copy_op != nullptr) { + auto kv = unique_grads.find(entry.sum); + if (kv == unique_grads.end()) { + unique_grads.emplace(std::move(entry.sum), std::make_pair(1, counter)); + } else { + NodePtr copy_node = Node::Create(); + std::ostringstream os; + os << entry.sum.node->attrs.name << "_" << kv->second.first << "_copy"; + kv->second.first++; + copy_node->attrs.op = copy_op; + copy_node->attrs.name = os.str(); + copy_node->inputs.emplace_back(entry.sum); + if (copy_node->attrs.op->attr_parser != nullptr) { + copy_node->attrs.op->attr_parser(&(copy_node->attrs)); + } + unique_grads.emplace(NodeEntry{std::move(copy_node), 0, 0}, std::make_pair(1, counter)); + } + } else { + ret.outputs[counter] = entry.sum; + } + ++counter; + } + if (copy_op != nullptr) { + for (const auto& kv : unique_grads) { + ret.outputs[kv.second.second] = kv.first; + } + } + return ret; +} + +// register pass +NNVM_REGISTER_PASS(MXGradient) +.describe("Return a gradient graph of src.attrs[\"ys\"] wrt src.attrs[\"xs\"]") +.set_body(Gradient) +.set_change_graph(true) +.depend_graph_attr("grad_ys") +.depend_graph_attr("grad_xs") +.depend_graph_attr("grad_ys_out_grad"); + +} // namespace +} // namespace pass +} // namespace nnvm diff --git a/src/nnvm/graph_algorithm.h b/src/nnvm/graph_algorithm.h new file mode 100644 index 000000000000..d1590c3b9846 --- /dev/null +++ b/src/nnvm/graph_algorithm.h @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2016 by Contributors + * \file graph_algorithm.h + * \brief This header contains graph algorithms on StaticGraph. + * It is used compute informations such as whether two + * operations can run in parallel, and helps allocation. +*/ +#ifndef MXNET_NNVM_GRAPH_ALGORITHM_H_ +#define MXNET_NNVM_GRAPH_ALGORITHM_H_ + +#include +#include + +namespace nnvm { +namespace pass { + +/*! + * \brief Find best path in the DAG, with reward defined + * by sum of reward of each node along the path. + * \param graph the original static graph. + * \param topo_order topo order of the nodes in the graph. + * \param node_reward the reward of each node. + * \param path the output path of nodes. + * \return the total reward of best path. + */ +inline uint32_t FindBestPath( + const IndexedGraph& graph, + const std::vector& node_reward, + std::vector* path) { + const uint32_t num_nodes = static_cast(graph.num_nodes()); + CHECK_EQ(num_nodes, node_reward.size()); + + std::vector best_reward(node_reward.size(), 0); + std::vector next_node(node_reward.size(), num_nodes); + uint32_t best_solution = 0, best_start_node = 0; + + // traverse in reverse topo order + for (uint32_t i = static_cast(graph.num_nodes()); i != 0; --i) { + const uint32_t nid = i - 1; + best_reward[nid] += node_reward[nid]; + if (best_reward[nid] > best_solution) { + best_solution = best_reward[nid]; + best_start_node = nid; + } + for (const auto& e : graph[nid].inputs) { + const uint32_t prev = e.node_id; + if (best_reward[nid] > best_reward[prev]) { + best_reward[prev] = best_reward[nid]; + next_node[prev] = nid; + } + } + } + path->clear(); + uint32_t reward = 0; + for (uint32_t nid = best_start_node; nid < num_nodes; nid = next_node[nid]) { + path->push_back(nid); reward += node_reward[nid]; + } + CHECK_EQ(reward, best_solution); + return best_solution; +} + +/*! + * \brief Color the nodes in the graph into index. + * The coloring algorithm tries to assign node group + * such that node in the same group cannot run in parallel. + * + * \param graph the original indexed graph. + * \param node_importance The importance of the node + * \param max_ncolor maximum number of colors allowed. + * \param color the color index of each of the node. + * \return the total number of colors. + */ +inline uint32_t ColorNodeGroup( + const IndexedGraph &graph, + std::vector node_importance, + uint32_t max_ncolor, + std::vector *color) { + CHECK_NE(max_ncolor, 0U); + CHECK_EQ(graph.num_nodes(), node_importance.size()); + + color->clear(); + color->resize(graph.num_nodes(), max_ncolor); + uint32_t cindex; + // greedy algorithm, every time + // find a path with best reward and assign a new color + // All the nodes in the path cannot run in parallel. + for (cindex = 0; cindex < max_ncolor - 1; ++cindex) { + std::vector path; + uint32_t reward = FindBestPath(graph, node_importance, &path); + if (reward == 0) break; + for (uint32_t nid : path) { + if (node_importance[nid] != 0) { + CHECK_EQ(color->at(nid), max_ncolor); + color->at(nid) = cindex; + // make the importance 0 after color is decided. + node_importance[nid] = 0; + } + } + } + // assign i for rest of the node + for (uint32_t i = 0; i < graph.num_nodes(); ++i) { + if (color->at(i) == max_ncolor) { + color->at(i) = cindex; + } + } + return cindex + 1; +} + +} // namespace pass +} // namespace nnvm + +#endif // MXNET_NNVM_GRAPH_ALGORITHM_H_ diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index 4ab777b6adb7..16ad0053e29a 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -217,12 +217,12 @@ bool OpPropInferAttr(const NodeAttrs& attrs, } bool OpPropInferShape(const NodeAttrs& attrs, - std::vector *iattr, - std::vector *oattr) { + mxnet::ShapeVector *iattr, + mxnet::ShapeVector *oattr) { auto finfer = [](const OperatorProperty* op, - std::vector *in, - std::vector *out, - std::vector *aux) { + mxnet::ShapeVector *in, + mxnet::ShapeVector *out, + mxnet::ShapeVector *aux) { return op->InferShape(in, out, aux); }; return OpPropInferAttr(attrs, iattr, oattr, finfer); @@ -294,23 +294,23 @@ std::vector > OpPropInplaceOption(const NodeAttrs& attrs) { } std::vector OpPropResourceRequest(const NodeAttrs& attrs) { - std::vector ishape; + mxnet::ShapeVector ishape; auto& prop = nnvm::get(attrs.parsed); return prop.ptr->ForwardResource(ishape); } std::vector OpBackResourceRequest(const NodeAttrs& attrs) { - std::vector ishape; + mxnet::ShapeVector ishape; auto& prop = nnvm::get(attrs.parsed); return prop.ptr->BackwardResource(ishape); } OpStatePtr OpPropCreateLayerOp(const NodeAttrs& attrs, Context ctx, - const std::vector& ishape, + const mxnet::ShapeVector& ishape, const std::vector& itype) { auto& prop = nnvm::get(attrs.parsed); - std::vector is(ishape.begin(), ishape.begin() + prop.arguments.size()); + mxnet::ShapeVector is(ishape.begin(), ishape.begin() + prop.arguments.size()); std::vector it(itype.begin(), itype.begin() + prop.arguments.size()); return OpStatePtr::Create(prop.ptr->CreateOperatorEx(ctx, &is, &it), prop.ptr.get()); @@ -452,7 +452,7 @@ void RegisterLegacyOpProp() { op.set_attr("FListInputNames", OpPropListInputNames); op.set_attr("FListOutputNames", OpPropListOutputNames); op.set_attr("FNumVisibleOutputs", OpPropNumVisibleOutputs); - op.set_attr("FInferShape", OpPropInferShape); + op.set_attr("FInferShape", OpPropInferShape); op.set_attr("FInferType", OpPropInferType); op.set_attr("FMutateInputs", OpPropMutateInputs); op.set_attr("FInplaceOption", OpPropInplaceOption); diff --git a/src/nnvm/plan_memory.cc b/src/nnvm/plan_memory.cc new file mode 100644 index 000000000000..2b18f990c845 --- /dev/null +++ b/src/nnvm/plan_memory.cc @@ -0,0 +1,412 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2016 by Contributors + * \file plan_memory.cc + * \brief Assign memory tag to each of the data entries. + */ +#include +#include +#include +#include +#include +#include +#include +#include "graph_algorithm.h" + +namespace nnvm { +namespace pass { + +namespace { + using namespace nnvm::top; +// Return bytes of data flag. +static int GetDTypeSize(int type_flag) { + switch (type_flag) { + case kUint8: + case kInt8: + return 1; + case kFloat16: + case kInt16: + case kUint16: + return 2; + case kFloat32: + case kInt32: + case kUint32: + return 4; + case kFloat64: + case kInt64: + case kUint64: + return 8; + default: + LOG(FATAL) << "unknown type_flag=" << type_flag; + return -1; + } +} + +// simple graph based allocator. +class GraphAllocator { + public: + // storage id equals integer. + using StorageID = int; + + // bad storage id + static const StorageID kBadStorageID = -1; + // external storage id + static const StorageID kExternalStorageID = -2; + // dynamic storage id + static const StorageID kDynamicStorageID = -3; + + // request a free storage + StorageID Request(int dev_id, int dtype, mxnet::TShape shape, uint32_t node_id) { + if (shape.ndim() == 0) return kBadStorageID; + // search memory block in [size / match_range_, size * match_range_) + // TODO(tqchen) add size of the dtype, assume 4 bytes for now + size_t size = shape.Size() * 4; + if (match_range_ == 0) return this->Alloc(dev_id, size); + auto begin = free_.lower_bound(size / match_range_); + auto mid = free_.lower_bound(size); + auto end = free_.upper_bound(size * match_range_); + // search for memory blocks larger than requested + for (auto it = mid; it != end; ++it) { + StorageEntry *e = it->second; + if (e->device_id != dev_id) continue; + if (node_color_.size() != 0 && + node_color_[e->released_by_node] != node_color_[node_id]) continue; + // Use exect matching strategy + e->max_bytes = std::max(size, e->max_bytes); + // find a exact match, erase from map and return + free_.erase(it); + return e->id; + } + // then search for memory blocks smaller than requested space + for (auto it = mid; it != begin;) { + --it; + StorageEntry *e = it->second; + if (e->device_id != dev_id) continue; + if (node_color_.size() != 0 && + node_color_[e->released_by_node] != node_color_[node_id]) continue; + // Use exect matching strategy + e->max_bytes = std::max(size, e->max_bytes); + // erase from map and return + free_.erase(it); + return e->id; + } + // cannot find anything return a new one. + return this->Alloc(dev_id, size); + } + // release a memory space. + void Release(StorageID id, uint32_t node_id) { + CHECK_NE(id, kBadStorageID); + if (id == kExternalStorageID || id == kDynamicStorageID) return; + StorageEntry *e = data_[id].get(); + e->released_by_node = node_id; + free_.insert({e->max_bytes, e}); + } + + // totoal number of bytes allocated + size_t TotalAllocBytes() const { + size_t total = 0; + for (auto &p : data_) { + total += p->max_bytes; + } + return total; + } + + // constructor + explicit GraphAllocator(const IndexedGraph* idx, const size_t match_range) : idx_(idx) { + this->Init(match_range, dmlc::GetEnv("NNVM_EXEC_NUM_TEMP", 1)); + } + + private: + // initialize the graph allocator + void Init(const size_t match_range, const uint32_t num_match_color) { + match_range_ = match_range; + num_match_color_ = num_match_color; + if (num_match_color_ > 1) { + std::vector importance(idx_->num_nodes(), 0); + for (uint32_t nid = 0; nid < idx_->num_nodes(); ++nid) { + if ((*idx_)[nid].source->is_variable()) continue; + importance[nid] = 1; + } + num_match_color_ = pass::ColorNodeGroup( + *idx_, importance, num_match_color_, &node_color_); + } + } + + StorageID Alloc(int dev_id, size_t size) { + StorageID id = static_cast(data_.size()); + std::unique_ptr ptr(new StorageEntry()); + ptr->id = id; + ptr->device_id = dev_id; + ptr->max_bytes = size; + data_.emplace_back(std::move(ptr)); + return id; + } + // internal storage entry + struct StorageEntry { + // the id of the entry. + StorageID id; + // the device id of the storage. + int device_id; + // maximum size of storage requested. + size_t max_bytes{0}; + // node index that released it last time + uint32_t released_by_node{0}; + }; + // scale used for rough match + size_t match_range_; + // whether use color based match algorithm + uint32_t num_match_color_{1}; + // the size of each dtype + std::vector dtype_size_dict_; + // free list of storage entry + std::multimap free_; + // all the storage resources available + std::vector > data_; + // color of nodes in the graph, used for auxiliary policy making. + std::vector node_color_; + // internal indexed graph + const IndexedGraph* idx_; +}; + +/* + * Internal method to perform the memory allocation for a graph + * */ +size_t AllocMemory(const Graph& ret, const IndexedGraph& idx, + const std::pair& node_range, + StorageVector* storage_ptr, + std::vector* storage_inplace_index_ptr, + const std::vector& entry_ref_count, + GraphAllocator* allocator) { + static auto& finplace_option = Op::GetAttr("FInplaceOption"); + static auto& finplace_identity = Op::GetAttr("FInplaceIdentity"); + static auto& fignore_inputs = Op::GetAttr("FIgnoreInputs"); + + // Get reference + auto &storage = *storage_ptr; + auto &storage_inplace_index = *storage_inplace_index_ptr; + + // Get attributes from the graph + const mxnet::ShapeVector& shape_vec = ret.GetAttr("shape"); + const DTypeVector& dtype_vec = ret.GetAttr("dtype"); + const DeviceVector* device_vec = nullptr; + + if (ret.attrs.count("device") != 0) { + device_vec = &(ret.GetAttr("device")); + } + size_t num_not_allocated = 0; + std::vector storage_ref_count(idx.num_node_entries(), 0); + + for (uint32_t nid = node_range.first; nid < node_range.second; ++nid) { + const auto& inode = idx[nid]; + if (inode.source->is_variable()) continue; + // check inplace option + if (finplace_option.count(inode.source->op()) != 0) { + auto inplace_pairs = finplace_option[inode.source->op()](inode.source->attrs); + std::vector identity; + if (finplace_identity.count(inode.source->op()) != 0) { + identity = finplace_identity[inode.source->op()](inode.source->attrs); + CHECK_EQ(identity.size(), inplace_pairs.size()) + << "FInplaceOption and FInplaceIdentity returned vectors of different " + << "size for operator " << inode.source->op()->name; + } else { + identity = std::vector(inplace_pairs.size(), false); + } + std::vector taken(inode.inputs.size(), false); + for (size_t ipair = 0; ipair < inplace_pairs.size(); ++ipair) { + const auto& kv = inplace_pairs[ipair]; + uint32_t eid_out = idx.entry_id(nid, kv.second); + uint32_t eid_in = idx.entry_id(inode.inputs[kv.first]); + auto sid_out = storage[eid_out]; + auto sid_in = storage[eid_in]; + bool ignore_all_inputs = (fignore_inputs.count(inode.source->op()) != 0 && + fignore_inputs[inode.source->op()]( + inode.source->attrs).size() == inode.source->num_inputs()); + if (taken[kv.first] == false && + sid_out == GraphAllocator::kBadStorageID && + sid_in >= 0 && + ((storage_ref_count[sid_in] == 1 && !ignore_all_inputs) || identity[ipair]) && + entry_ref_count[eid_out] > 0 && + shape_vec[eid_out].Size() == shape_vec[eid_in].Size() && + (dtype_vec[eid_out] == dtype_vec[eid_in] || + GetDTypeSize(dtype_vec[eid_out]) == GetDTypeSize(dtype_vec[eid_in]))) { + // inplace optimization + taken[kv.first] = true; + storage[eid_out] = sid_in; + // Reuse storage for output and add ref count of output + // to storage. This will get substracted later in free + // input section. + storage_ref_count[sid_in] += entry_ref_count[eid_out]; + storage_inplace_index[eid_out] = kv.first; + } + } + } + // normal allocation + const int dev_id = (device_vec != nullptr) ? device_vec->at(nid) : 0; + // sort output nodes based on size before allocating output + std::multimap eids; + for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) { + uint32_t eid = idx.entry_id(nid, index); + // only request memory for kBadStorageID + if (storage[eid] == GraphAllocator::kBadStorageID) { + auto &eshape = shape_vec[eid]; + size_t esize = 0; + if (eshape.ndim() != 0) esize = eshape.Size(); + eids.insert(std::make_pair(esize, eid)); + } + } + for (auto rit = eids.rbegin(); rit != eids.rend(); ++rit) { + uint32_t eid = rit->second; + auto sid = allocator->Request(dev_id, dtype_vec[eid], shape_vec[eid], nid); + if (sid >= 0) { + storage_ref_count[sid] = entry_ref_count[eid]; + } + storage[eid] = sid; + } + // check if certain inputs is ignored. + std::vector ignore_inputs; + if (fignore_inputs.count(inode.source->op()) != 0) { + ignore_inputs = fignore_inputs[inode.source->op()](inode.source->attrs); + std::sort(ignore_inputs.begin(), ignore_inputs.end()); + } + // then free inputs + for (size_t i = 0; i < inode.inputs.size(); ++i) { + // ref counter of ignored input is already decreased. + if (std::binary_search(ignore_inputs.begin(), ignore_inputs.end(), i)) continue; + const auto& e = inode.inputs[i]; + uint32_t eid = idx.entry_id(e); + auto sid = storage[eid]; + // storage_ref_count == 0 means it is taken by inplace op + if (sid < 0) continue; + // if we decrease it to zero, means we are ready to relase + --storage_ref_count[sid]; + if (storage_ref_count[sid] == 0) { + allocator->Release(sid, nid); + } + } + // check if there are outputs that can be freeded immediately + // these output are not referenced by any operator. + for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) { + uint32_t eid = idx.entry_id(nid, index); + auto sid = storage[eid]; + if (sid >= 0 && storage_ref_count[sid] == 0) { + allocator->Release(sid, nid); + // use -2 to indicate that the node was never touched. + storage_inplace_index[eid] = -2; + } + if (storage[eid] == GraphAllocator::kBadStorageID) { + ++num_not_allocated; + } + } + } + return num_not_allocated; +} + + +// function to plan memory +Graph PlanMemory(Graph ret) { + // setup ref counter + const IndexedGraph& idx = ret.indexed_graph(); + static auto& fignore_inputs = Op::GetAttr("FIgnoreInputs"); + std::pair node_range = {0, idx.num_nodes()}; + if (ret.attrs.count("node_range")) { + node_range = ret.MoveCopyAttr >("node_range"); + } + // reference counter of each node + std::vector ref_count; + // step 1: initialize reference count + if (ret.attrs.count("ref_count") != 0) { + ref_count = ret.MoveCopyAttr >("ref_count"); + } else { + ref_count.resize(idx.num_node_entries(), 0); + for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) { + const auto& inode = idx[nid]; + if (inode.source->is_variable()) continue; + for (const auto& e : inode.inputs) { + ++ref_count[idx.entry_id(e)]; + } + // no dataflow dependency is needed for those are ignored. + // revoke the dependency counter. + if (fignore_inputs.count(inode.source->op()) != 0) { + auto ignore_inputs = fignore_inputs[inode.source->op()](inode.source->attrs); + for (uint32_t i : ignore_inputs) { + --ref_count[idx.entry_id(inode.inputs[i])]; + } + } + } + for (const auto& e : idx.outputs()) { + ++ref_count[idx.entry_id(e)]; + } + } + // step 2: allocate memory. + StorageVector storage; + if (ret.attrs.count("storage") != 0) { + storage = ret.MoveCopyAttr("storage"); + } else { + storage.resize(idx.num_node_entries(), -1); + } + + // Search the best NNVM_EXEC_MATCH_RANGE parameter. This is turned off by default + size_t min_allocated_bytes = -1; + size_t max_match_range = dmlc::GetEnv("NNVM_EXEC_MATCH_RANGE", 16); + size_t min_match_range = + dmlc::GetEnv("NNVM_AUTO_SEARCH_MATCH_RANGE", false) ? 1 : max_match_range; + for (size_t match_range = min_match_range; match_range <= max_match_range; match_range *= 2) { + // Make a copy of related fields + StorageVector storage_vec(storage); + std::vector storage_inplace_index(idx.num_node_entries(), -1); + + // the allocator + GraphAllocator allocator(&idx, match_range); + + // number of entries that are not statically allocated. + size_t storage_num_not_allocated = + AllocMemory(ret, idx, node_range, &storage_vec, &storage_inplace_index, + ref_count, &allocator); + size_t storage_allocated_bytes = allocator.TotalAllocBytes(); + + // Choose the plan which leads to minimal memory usage + if (min_allocated_bytes > storage_allocated_bytes) { + ret.attrs["storage_id"] = std::make_shared(std::move(storage_vec)); + ret.attrs["storage_inplace_index"] = std::make_shared(std::move(storage_inplace_index)); + ret.attrs["storage_allocated_bytes"] = std::make_shared(storage_allocated_bytes); + ret.attrs["storage_num_not_allocated"] = std::make_shared(storage_num_not_allocated); + min_allocated_bytes = storage_allocated_bytes; + } + + if (max_match_range == 0) { + break; + } + } + return ret; +} + +NNVM_REGISTER_PASS(MXPlanMemory) +.describe("Plan the memory allocation of each node entries.") +.set_body(PlanMemory) +.set_change_graph(false) +.depend_graph_attr("dtype") +.depend_graph_attr("shape") +.provide_graph_attr("storage_id") +.provide_graph_attr("storage_inplace_index"); + +} // namespace +} // namespace pass +} // namespace nnvm diff --git a/src/operator/batch_norm_v1-inl.h b/src/operator/batch_norm_v1-inl.h index f4116e30186e..f407a5cce61b 100644 --- a/src/operator/batch_norm_v1-inl.h +++ b/src/operator/batch_norm_v1-inl.h @@ -255,15 +255,15 @@ class BatchNormV1Prop : public OperatorProperty { return param_.__DICT__(); } - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { + bool InferShape(mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape, + mxnet::ShapeVector *aux_shape) const override { using namespace mshadow; CHECK_EQ(in_shape->size(), 3U) << "Input:[data, gamma, beta]"; - const TShape &dshape = in_shape->at(0); + const mxnet::TShape &dshape = in_shape->at(0); if (dshape.ndim() == 0) return false; - in_shape->at(1) = TShape(Shape1(dshape[1])); - in_shape->at(2) = TShape(Shape1(dshape[1])); + in_shape->at(1) = mxnet::TShape(Shape1(dshape[1])); + in_shape->at(2) = mxnet::TShape(Shape1(dshape[1])); out_shape->clear(); out_shape->push_back(dshape); out_shape->push_back(Shape1(dshape[1])); @@ -331,7 +331,7 @@ class BatchNormV1Prop : public OperatorProperty { } std::vector BackwardResource( - const std::vector &in_shape) const override { + const mxnet::ShapeVector &in_shape) const override { return {ResourceRequest::kTempSpace}; } @@ -363,7 +363,7 @@ class BatchNormV1Prop : public OperatorProperty { return NULL; } - Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + Operator* CreateOperatorEx(Context ctx, mxnet::ShapeVector *in_shape, std::vector *in_type) const override; inline const BatchNormV1Param& getParam() const { diff --git a/src/operator/batch_norm_v1.cc b/src/operator/batch_norm_v1.cc index 2d19107eda1e..dc9f724aff18 100644 --- a/src/operator/batch_norm_v1.cc +++ b/src/operator/batch_norm_v1.cc @@ -35,9 +35,9 @@ Operator *CreateOp(BatchNormV1Param param, int dtype) { } // DO_BIND_DISPATCH comes from operator_common.h -Operator *BatchNormV1Prop::CreateOperatorEx(Context ctx, std::vector *in_shape, +Operator *BatchNormV1Prop::CreateOperatorEx(Context ctx, mxnet::ShapeVector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; + mxnet::ShapeVector out_shape, aux_shape; std::vector out_type, aux_type; CHECK(InferType(in_type, &out_type, &aux_type)); CHECK(InferShape(in_shape, &out_shape, &aux_shape)); diff --git a/src/operator/bilinear_sampler-inl.h b/src/operator/bilinear_sampler-inl.h index 49a5b5e5d5d8..8b1ff38709b6 100644 --- a/src/operator/bilinear_sampler-inl.h +++ b/src/operator/bilinear_sampler-inl.h @@ -142,13 +142,13 @@ class BilinearSamplerProp : public OperatorProperty { return param_.__DICT__(); } - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { + bool InferShape(mxnet::ShapeVector *in_shape, + mxnet::ShapeVector *out_shape, + mxnet::ShapeVector *aux_shape) const override { using namespace mshadow; CHECK_EQ(in_shape->size(), 2U) << "Input:[data, grid]"; - const TShape &dshape = (*in_shape)[bs::kData]; - const TShape &lshape = (*in_shape)[bs::kGrid]; + const mxnet::TShape &dshape = (*in_shape)[bs::kData]; + const mxnet::TShape &lshape = (*in_shape)[bs::kGrid]; if (dshape.ndim() == 0) return false; CHECK_EQ(dshape.ndim(), 4U) \ << "input data should be 4D in batch-num_filter-y-x"; @@ -226,7 +226,7 @@ class BilinearSamplerProp : public OperatorProperty { return NULL; } - Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + Operator* CreateOperatorEx(Context ctx, mxnet::ShapeVector *in_shape, std::vector *in_type) const override; private: diff --git a/src/operator/bilinear_sampler.cc b/src/operator/bilinear_sampler.cc index c435fdeca481..a2442bffa232 100644 --- a/src/operator/bilinear_sampler.cc +++ b/src/operator/bilinear_sampler.cc @@ -170,7 +170,7 @@ Operator* CreateOp(BilinearSamplerParam param, int dtype) { return op; } -Operator *BilinearSamplerProp::CreateOperatorEx(Context ctx, std::vector *in_shape, +Operator *BilinearSamplerProp::CreateOperatorEx(Context ctx, mxnet::ShapeVector *in_shape, std::vector *in_type) const { DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/contrib/adamw-inl.h b/src/operator/contrib/adamw-inl.h index 3d76b33ae765..07feaefe87aa 100644 --- a/src/operator/contrib/adamw-inl.h +++ b/src/operator/contrib/adamw-inl.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "../operator_common.h" #include "../mshadow_op.h" #include "../elemwise_op_common.h" @@ -48,7 +49,6 @@ struct AdamWParam : public dmlc::Parameter { float epsilon; float wd; float eta; - float rescale_grad; float clip_gradient; DMLC_DECLARE_PARAMETER(AdamWParam) { DMLC_DECLARE_FIELD(lr) @@ -69,9 +69,6 @@ struct AdamWParam : public dmlc::Parameter { "The penalty scales with the square of the magnitude of each weight."); DMLC_DECLARE_FIELD(eta) .describe("Learning rate schedule multiplier"); - DMLC_DECLARE_FIELD(rescale_grad) - .set_default(1.0f) - .describe("Rescale gradient to grad = rescale_grad*grad."); DMLC_DECLARE_FIELD(clip_gradient) .set_default(-1.0f) .describe("Clip gradient to the range of [-clip_gradient, clip_gradient] " @@ -80,44 +77,138 @@ struct AdamWParam : public dmlc::Parameter { } }; +// rescale_grad is a reserved argument at position -1. Example: +// n_in = 2: weight, grad (fp16) +// n_out = 1: weight (fp16) +// total_in = 6: weight, grad, mean, var, weight32, rescale_grad (fp32) +template +inline bool MPUpdateInferShape(const nnvm::NodeAttrs& attrs, + mxnet::ShapeVector *in_attrs, + mxnet::ShapeVector *out_attrs) { + CHECK_EQ(in_attrs->size(), static_cast(total_in)) << " in operator " << attrs.name; + CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + // rescale_grad.shape = (1,) + SHAPE_ASSIGN_CHECK(*in_attrs, total_in - 1, mshadow::Shape1(1)); + return ElemwiseAttr( + attrs, in_attrs, out_attrs, mxnet::TShape()); +} + +// rescale_grad is a reserved argument at position -1. Example: +// n_in = 2: weight, grad (fp16) +// n_out = 1: weight (fp16) +// total_in = 6: weight, grad, mean, var, weight32, rescale_grad (fp32) +template +inline bool MPUpdateInferType(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), static_cast(total_in)) << " in operator " << attrs.name; + CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + for (int i = n_in; i < total_in; ++i) { + TYPE_ASSIGN_CHECK(*in_attrs, i, mshadow::kFloat32); + } + return ElemwiseAttr( + attrs, in_attrs, out_attrs, -1); +} + +template +struct MPAdamWKernel { + template + MSHADOW_XINLINE static void Map(int i, DType* out_data, float* mean_data, + float* var_data, const DType* weight_data, const DType* grad_data, float* weight32, + const float param_clip_gradient, const float param_beta1, const float param_beta2, + const float param_eta, const float param_lr, const float param_wd, + const float param_rescale_grad, const float param_epsilon) { + float w = weight32[i]; + float mean = mean_data[i]; + float var = var_data[i]; + float scaled_grad = param_rescale_grad*static_cast(grad_data[i]); + if (param_clip_gradient >= 0.0f) { + mean = param_beta1 * mean + + (1 - param_beta1) * mshadow_op::clip::Map(scaled_grad, param_clip_gradient); + var = param_beta2 * var + (1 - param_beta2) * + mshadow_op::square::Map(mshadow_op::clip::Map(scaled_grad, param_clip_gradient)); + } else { + mean = param_beta1 * mean + (1 - param_beta1) * scaled_grad; + var = param_beta2 * var + (1 - param_beta2) * mshadow_op::square::Map(scaled_grad); + } + mean_data[i] = mean; + var_data[i] = var; + w = w - param_eta * (param_lr * mean / (mshadow_op::square_root::Map(var) + param_epsilon) + + param_wd * w); + weight32[i] = w; + KERNEL_ASSIGN(out_data[i], req, w); + } +}; + + +template +struct MPAdamWUpdate { + static inline void Forward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs, + const float rescale_grad) { + using namespace mxnet_op; + AdamWParam param = nnvm::get(attrs.parsed); + Stream* s = ctx.get_stream(); + MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { + Tensor weight = inputs[0].FlatTo2D(s); + Tensor grad = inputs[1].FlatTo2D(s); + Tensor mean = inputs[2].FlatTo2D(s); + Tensor var = inputs[3].FlatTo2D(s); + Tensor weight32 = inputs[4].FlatTo2D(s); + Tensor out = outputs[0].FlatTo2D(s); + MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, { + Kernel, xpu>::Launch(s, weight.shape_.Size(), out.dptr_, mean.dptr_, + var.dptr_, weight.dptr_, grad.dptr_, weight32.dptr_, param.clip_gradient, param.beta1, + param.beta2, param.eta, param.lr, param.wd, rescale_grad, param.epsilon); + }); + }); + } +}; + /* * \brief adam_w update. */ template -inline void AdamWUpdate(const nnvm::NodeAttrs& attrs, - const OpContext &ctx, - const std::vector &inputs, - const std::vector &req, - const std::vector &outputs) { - using namespace mshadow; - using namespace mshadow::expr; - using namespace mshadow_op; - const AdamWParam& param = nnvm::get(attrs.parsed); - Stream* s = ctx.get_stream(); - MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { - Tensor weight = inputs[0].FlatTo2D(s); - Tensor grad = inputs[1].FlatTo2D(s); - Tensor mean = inputs[2].FlatTo2D(s); - Tensor var = inputs[3].FlatTo2D(s); - Tensor out = outputs[0].FlatTo2D(s); +struct AdamWUpdate { + static inline void Forward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs, + const float rescale_grad) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace mshadow_op; + const AdamWParam& param = nnvm::get(attrs.parsed); + Stream* s = ctx.get_stream(); + MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { + Tensor weight = inputs[0].FlatTo2D(s); + Tensor grad = inputs[1].FlatTo2D(s); + Tensor mean = inputs[2].FlatTo2D(s); + Tensor var = inputs[3].FlatTo2D(s); + Tensor out = outputs[0].FlatTo2D(s); - grad = scalar(param.rescale_grad) * grad; - if (param.clip_gradient >= 0.0f) { - mean = scalar(param.beta1)*mean + scalar(1.f-param.beta1) * - F(grad, DType(param.clip_gradient)); - var = scalar(param.beta2)*var + scalar(1.f-param.beta2)*F( - F(grad, DType(param.clip_gradient))); - } else { - mean = scalar(param.beta1)*mean + scalar(1.f-param.beta1) * grad; - var = scalar(param.beta2)*var + scalar(1.f-param.beta2) * F(grad); - } - Assign(out, req[0], - weight - - scalar(param.eta) * (scalar(param.lr) * - mean / (F(var) + scalar(param.epsilon)) + - (scalar(param.wd) * weight))); - }); -} + grad = scalar(rescale_grad) * grad; + if (param.clip_gradient >= 0.0f) { + mean = scalar(param.beta1)*mean + scalar(1.f-param.beta1) * + F(grad, DType(param.clip_gradient)); + var = scalar(param.beta2)*var + scalar(1.f-param.beta2)*F( + F(grad, DType(param.clip_gradient))); + } else { + mean = scalar(param.beta1)*mean + scalar(1.f-param.beta1) * grad; + var = scalar(param.beta2)*var + scalar(1.f-param.beta2) * F(grad); + } + Assign(out, req[0], + weight - + scalar(param.eta) * (scalar(param.lr) * + mean / (F(var) + scalar(param.epsilon)) + + (scalar(param.wd) * weight))); + }); + } +}; } // namespace op } // namespace mxnet diff --git a/src/operator/contrib/adamw.cc b/src/operator/contrib/adamw.cc index 94623fe08a9e..f0716c6020f9 100644 --- a/src/operator/contrib/adamw.cc +++ b/src/operator/contrib/adamw.cc @@ -24,13 +24,78 @@ * \author Haibin Lin */ #include "./adamw-inl.h" +#include "../optimizer_op-inl.h" namespace mxnet { namespace op { DMLC_REGISTER_PARAMETER(AdamWParam); -NNVM_REGISTER_OP(_contrib_adamw_update) +template