Skip to content

Commit 39d8a6a

Browse files
eric-haibin-linEC2 Default UserLin
authored and
chinakook
committed
Switch to cached op in the testing suite (apache#18579)
* add default ctx to cachedop fwd * add test * perl fix * initial commit * update sparse tests * add aux_states * fix aux-state type * fix some tests * fix check symbolic forwrad/backward * fix symbolic grad check * arg_dict fixes * support init ops * support forward only graph * fix check symbolic backward stype * add missing file * replace extension test bind * replace bind with _bind * simplify backward_mul implementation * small fix * drop contrib.sparseembedding * remove simple_bind in test sparse ops * use simple_bind * replave simple bind in quantization * fix aux index * update amp simple_bind calls * drop ifft * fix a bug found in subgraph op * add aux_array method * replace symbols * minor fix * fix executor default context * fix import * bug fix for nd.where * add subgraph test * fix forward grad req * fix batch dot dtype * remove unused code * fix slice dtype * fix attach grad * remove tests for non-existing sparse ops * MXCachedOpGetOptimizedSymbol * fix foreach test * enhance err msg * skip failed test * add docs * add docs * fix lint * fix lint, remove quantization * fix lint * fix lint * fix lint * fix build and import * fix import * fix perl call * fix test * remove perl binding * remove reshape test * fix profiler, trt * remove tensorrt test * remove quantization tests * fix import * fix conflcit * fix lint * skip buggy test Co-authored-by: EC2 Default User <[email protected]> Co-authored-by: Lin <[email protected]>
1 parent 15c6079 commit 39d8a6a

File tree

260 files changed

+852
-58459
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

260 files changed

+852
-58459
lines changed

‎cd/mxnet_lib/mxnet_lib_pipeline.groovy

-21
Original file line numberDiff line numberDiff line change
@@ -42,16 +42,6 @@ def get_pipeline(mxnet_variant, build_fn) {
4242
}
4343
}
4444

45-
if (mxnet_variant.startsWith('cu')) {
46-
tests["${mxnet_variant}: Quantization Python 3"] = {
47-
stage("${mxnet_variant}: Quantization Python 3") {
48-
timeout(time: max_time, unit: 'MINUTES') {
49-
test_gpu_quantization_py3(mxnet_variant)
50-
}
51-
}
52-
}
53-
}
54-
5545
parallel tests
5646
}
5747

@@ -103,17 +93,6 @@ def unittest_py3(mxnet_variant) {
10393
}
10494
}
10595

106-
// Tests quantization in P3 instance using Python 3
107-
def test_gpu_quantization_py3(mxnet_variant) {
108-
node(NODE_LINUX_GPU_P3) {
109-
ws("workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}") {
110-
def image = get_environment(mxnet_variant)
111-
ci_utils.unpack_and_init("mxnet_${mxnet_variant}", get_stash(mxnet_variant), false)
112-
ci_utils.docker_run(image, "unittest_ubuntu_python3_quantization_gpu", true)
113-
}
114-
}
115-
}
116-
11796
// Pushes artifact to artifact repository
11897
def push(mxnet_variant) {
11998
node(NODE_LINUX_CPU) {

‎ci/docker/Dockerfile.build.ubuntu

-4
Original file line numberDiff line numberDiff line change
@@ -126,10 +126,6 @@ RUN update-java-alternatives -s java-1.8.0-openjdk-amd64
126126
COPY install/ubuntu_julia.sh /work/
127127
RUN /work/ubuntu_julia.sh
128128

129-
# PDL::CCS missing on 18.04
130-
COPY install/ubuntu_perl.sh /work/
131-
RUN /work/ubuntu_perl.sh
132-
133129
# MXNetJS nightly needs emscripten for wasm
134130
COPY install/ubuntu_emscripten.sh /work/
135131
RUN /work/ubuntu_emscripten.sh

‎ci/docker/Dockerfile.build.ubuntu_cpu_julia

-3
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,6 @@ RUN /work/ubuntu_scala.sh
3939
COPY install/ubuntu_clojure.sh /work/
4040
RUN /work/ubuntu_clojure.sh
4141

42-
COPY install/ubuntu_perl.sh /work/
43-
RUN /work/ubuntu_perl.sh
44-
4542
COPY install/ubuntu_julia.sh /work/
4643
RUN /work/ubuntu_julia.sh
4744

‎ci/docker/install/ubuntu_perl.sh

-27
This file was deleted.

‎ci/docker/runtime_functions.sh

-40
Original file line numberDiff line numberDiff line change
@@ -925,7 +925,6 @@ cd_unittest_ubuntu() {
925925

926926
pytest -m 'not serial' -n 4 --durations=50 --verbose tests/python/unittest
927927
pytest -m 'serial' --durations=50 --verbose tests/python/unittest
928-
pytest -n 4 --durations=50 --verbose tests/python/quantization
929928

930929
# https://github.com/apache/incubator-mxnet/issues/11801
931930
# if [[ ${mxnet_variant} = "cpu" ]] || [[ ${mxnet_variant} = "mkl" ]]; then
@@ -963,7 +962,6 @@ unittest_ubuntu_python3_cpu() {
963962
MXNET_ENGINE_TYPE=NaiveEngine \
964963
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
965964
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
966-
pytest -n 4 --durations=50 --cov-report xml:tests_quantization.xml --verbose tests/python/quantization
967965
}
968966

969967
unittest_ubuntu_python3_cpu_serial() {
@@ -976,7 +974,6 @@ unittest_ubuntu_python3_cpu_serial() {
976974
export MXNET_ENABLE_CYTHON=0
977975
export DMLC_LOG_STACK_TRACE_DEPTH=10
978976
pytest --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
979-
pytest --durations=50 --cov-report xml:tests_quantization.xml --verbose tests/python/quantization
980977
}
981978

982979
unittest_ubuntu_python3_cpu_mkldnn() {
@@ -1044,38 +1041,6 @@ unittest_ubuntu_python3_gpu_nocudnn() {
10441041
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
10451042
}
10461043

1047-
unittest_ubuntu_tensorrt_gpu() {
1048-
set -ex
1049-
export PYTHONPATH=./python/
1050-
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
1051-
export MXNET_SUBGRAPH_VERBOSE=0
1052-
export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH
1053-
export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
1054-
export MXNET_ENABLE_CYTHON=0
1055-
export DMLC_LOG_STACK_TRACE_DEPTH=10
1056-
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
1057-
pytest -n 4 --durations=50 --cov-report xml:tests_trt_gpu.xml --verbose --capture=no tests/python/tensorrt/test_ops.py
1058-
pytest -k 'not test_ops' --durations=50 --cov-report xml:tests_trt_gpu.xml --cov-append --verbose --capture=no tests/python/tensorrt/
1059-
}
1060-
1061-
# quantization gpu currently only runs on P3 instances
1062-
# need to separte it from unittest_ubuntu_python3_gpu()
1063-
unittest_ubuntu_python3_quantization_gpu() {
1064-
set -ex
1065-
if [ -f /etc/redhat-release ]; then
1066-
source /opt/rh/rh-python36/enable
1067-
fi
1068-
export PYTHONPATH=./python/
1069-
export MXNET_MKLDNN_DEBUG=0 # Ignored if not present
1070-
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
1071-
export MXNET_SUBGRAPH_VERBOSE=0
1072-
export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
1073-
export MXNET_ENABLE_CYTHON=0
1074-
export DMLC_LOG_STACK_TRACE_DEPTH=10
1075-
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
1076-
pytest -n 4 --durations=50 --cov-report xml:tests_quantization_gpu.xml --verbose tests/python/quantization_gpu
1077-
}
1078-
10791044
unittest_centos7_cpu_scala() {
10801045
set -ex
10811046
source /opt/rh/devtoolset-7/enable
@@ -1104,11 +1069,6 @@ unittest_ubuntu_cpu_clojure_integration() {
11041069
}
11051070

11061071

1107-
unittest_ubuntu_cpugpu_perl() {
1108-
set -ex
1109-
./perl-package/test.sh
1110-
}
1111-
11121072
unittest_cpp() {
11131073
set -ex
11141074
build/tests/mxnet_unit_tests

‎ci/jenkins/Jenkins_steps.groovy

-77
Original file line numberDiff line numberDiff line change
@@ -839,24 +839,6 @@ def test_unix_python3_gpu(lib_name) {
839839
}]
840840
}
841841

842-
def test_unix_python3_quantize_gpu(lib_name) {
843-
return ['Python3: Quantize GPU': {
844-
node(NODE_LINUX_GPU_P3) {
845-
ws('workspace/ut-python3-quantize-gpu') {
846-
timeout(time: max_time, unit: 'MINUTES') {
847-
try {
848-
utils.unpack_and_init(lib_name, mx_lib)
849-
utils.docker_run('ubuntu_gpu_cu101', 'unittest_ubuntu_python3_quantization_gpu', true)
850-
utils.publish_test_coverage()
851-
} finally {
852-
utils.collect_test_results_unix('tests_quantization_gpu.xml', 'tests_python3_quantize_gpu.xml')
853-
}
854-
}
855-
}
856-
}
857-
}]
858-
}
859-
860842
def test_unix_python3_debug_cpu() {
861843
return ['Python3: CPU debug': {
862844
node(NODE_LINUX_CPU) {
@@ -955,24 +937,6 @@ def test_unix_python3_mkldnn_nocudnn_gpu(lib_name) {
955937
}]
956938
}
957939

958-
def test_unix_python3_tensorrt_gpu(lib_name) {
959-
return ['Python3: TensorRT GPU': {
960-
node(NODE_LINUX_GPU_P3) {
961-
ws('workspace/build-tensorrt') {
962-
timeout(time: max_time, unit: 'MINUTES') {
963-
try {
964-
utils.unpack_and_init(lib_name, mx_tensorrt_lib)
965-
utils.docker_run('ubuntu_gpu_tensorrt', 'unittest_ubuntu_tensorrt_gpu', true)
966-
utils.publish_test_coverage()
967-
} finally {
968-
utils.collect_test_results_unix('tests_tensorrt.xml', 'tests_python3_tensorrt_gpu.xml')
969-
}
970-
}
971-
}
972-
}
973-
}]
974-
}
975-
976940
def test_unix_cpp_package_gpu(lib_name) {
977941
return ['cpp-package GPU Makefile': {
978942
node(NODE_LINUX_GPU_G4) {
@@ -1084,20 +1048,6 @@ def test_unix_r_mkldnn_cpu(lib_name) {
10841048
}]
10851049
}
10861050

1087-
def test_unix_perl_cpu(lib_name) {
1088-
return ['Perl: CPU Makefile': {
1089-
node(NODE_LINUX_CPU) {
1090-
ws('workspace/ut-perl-cpu') {
1091-
timeout(time: max_time, unit: 'MINUTES') {
1092-
utils.unpack_and_init(lib_name, mx_lib_make)
1093-
utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false)
1094-
utils.publish_test_coverage()
1095-
}
1096-
}
1097-
}
1098-
}]
1099-
}
1100-
11011051
def test_unix_cpp_gpu(lib_name) {
11021052
return ['Cpp: GPU': {
11031053
node(NODE_LINUX_GPU_G4) {
@@ -1126,20 +1076,6 @@ def test_unix_cpp_cpu(lib_name) {
11261076
}]
11271077
}
11281078

1129-
def test_unix_perl_gpu(lib_name) {
1130-
return ['Perl: GPU Makefile': {
1131-
node(NODE_LINUX_GPU_G4) {
1132-
ws('workspace/ut-perl-gpu') {
1133-
timeout(time: max_time, unit: 'MINUTES') {
1134-
utils.unpack_and_init(lib_name, mx_lib_make)
1135-
utils.docker_run('ubuntu_gpu_cu101', 'unittest_ubuntu_cpugpu_perl', true)
1136-
utils.publish_test_coverage()
1137-
}
1138-
}
1139-
}
1140-
}]
1141-
}
1142-
11431079
def test_unix_r_gpu(lib_name) {
11441080
return ['R: GPU': {
11451081
node(NODE_LINUX_GPU_G4) {
@@ -1312,19 +1248,6 @@ def test_centos7_python3_cd_gpu(lib_name) {
13121248
}]
13131249
}
13141250

1315-
def test_centos7_quantization_cd_gpu(lib_name) {
1316-
return ['Quantization Python3: CentOS 7 GPU CD': {
1317-
node(NODE_LINUX_GPU_P3) {
1318-
ws('workspace/test-cd-static/gpu') {
1319-
timeout(time: max_time, unit: 'MINUTES') {
1320-
utils.unpack_and_init(lib_name, mx_cd_lib)
1321-
utils.docker_run('centos7_gpu_cu102', 'unittest_ubuntu_python3_quantization_gpu', true)
1322-
}
1323-
}
1324-
}
1325-
}]
1326-
}
1327-
13281251
def test_centos7_pypi_package_cd_gpu(lib_name) {
13291252
return ['PyPI package: CentOS 7 GPU CD': {
13301253
node(NODE_LINUX_GPU) {

‎ci/jenkins/Jenkinsfile_centos_gpu

-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ core_logic: {
4242
utils.parallel_stage('Tests', [
4343
custom_steps.test_centos7_python3_gpu('centos7_gpu'),
4444
custom_steps.test_centos7_python3_cd_gpu('centos7_gpu_cd'),
45-
custom_steps.test_centos7_quantization_cd_gpu('centos7_gpu_cd'),
4645
custom_steps.test_centos7_pypi_package_cd_gpu('centos7_gpu_cd')
4746
])
4847
}

‎ci/jenkins/Jenkinsfile_unix_cpu

-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ core_logic: {
5454
custom_steps.test_unix_scala_mkldnn_cpu('mkldnn_cpu_make'),
5555
custom_steps.test_unix_clojure_cpu('cpu_make'),
5656
custom_steps.test_unix_clojure_integration_cpu('cpu_make'),
57-
custom_steps.test_unix_perl_cpu('cpu_make'),
5857
custom_steps.test_unix_r_cpu('cpu'),
5958
custom_steps.test_unix_r_mkldnn_cpu('mkldnn_cpu'),
6059
custom_steps.test_unix_julia07_cpu('cpu'),

‎ci/jenkins/Jenkinsfile_unix_gpu

-3
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,8 @@ core_logic: {
4848

4949
utils.parallel_stage('Tests', [
5050
custom_steps.test_unix_python3_gpu('gpu'),
51-
custom_steps.test_unix_python3_quantize_gpu('gpu'),
5251
custom_steps.test_unix_python3_mkldnn_gpu('mkldnn_gpu'),
5352
custom_steps.test_unix_python3_mkldnn_nocudnn_gpu('mkldnn_gpu_nocudnn'),
54-
custom_steps.test_unix_python3_tensorrt_gpu('tensorrt'),
55-
custom_steps.test_unix_perl_gpu('gpu_make'),
5653
custom_steps.test_unix_r_gpu('gpu'),
5754
custom_steps.test_unix_cpp_gpu('cmake_gpu'),
5855
custom_steps.test_unix_cpp_package_gpu('gpu_make'),

‎docs/python_docs/python/api/gluon/contrib/index.rst

+1-2
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ Neural Network
5050
Concurrent
5151
HybridConcurrent
5252
Identity
53-
SparseEmbedding
5453
SyncBatchNorm
5554
PixelShuffle1D
5655
PixelShuffle2D
@@ -165,4 +164,4 @@ API Reference
165164

166165
.. automodule:: mxnet.gluon.contrib.estimator
167166
:members:
168-
:imported-members:
167+
:imported-members:

‎docs/python_docs/python/tutorials/packages/ndarray/sparse/train_gluon.md

-4
Original file line numberDiff line numberDiff line change
@@ -465,10 +465,6 @@ Memory Allocation for Weight Gradient:
465465
0.000 MBs ( 0.050%) for fullyconnected3
466466
```
467467

468-
### Advanced: Sparse `weight`
469-
470-
You can optimize this example further by setting the weight's `stype` to `'row_sparse'`, but whether `'row_sparse'` weights make sense or not will depends on your specific task. See [contrib.SparseEmbedding](https://github.com/apache/incubator-mxnet/blob/master/python/mxnet/gluon/contrib/nn/basic_layers.py#L118) for an example of this.
471-
472468
## Conclusion
473469

474470
As part of this tutorial, we learned how to write sparse data to disk in LibSVM format and load it back in sparse batches with the [LibSVMIter](/api/python/docs/api/mxnet/io/index.html#mxnet.io.LibSVMIter). We learned how to improve the performance of Gluon's [nn.Dense](/api/python/docs/api/gluon/nn/index.html#mxnet.gluon.nn.Dense) on sparse arrays using `mx.nd.sparse`. And lastly, we set `grad_stype` to `'row_sparse'` to reduce the size of the gradient and speed up the parameter update step.

0 commit comments

Comments
 (0)