[v1.9.x] Port #20709 to v1.9.x branch

.licenserc.yaml

@@ -49,25 +49,30 @@ header:
     - 'cmake/Modules/FindJeMalloc.cmake'
     # files licensed under bsd 2-clause
     - 'example/ssd/dataset/pycocotools/coco.py'
+    # files licensed under bsd 2-clause + caffe
+    - 'src/operator/nn/pool.cuh'
+    - 'src/operator/nn/pool.h'
+    - 'src/operator/nn/im2col.cuh'
+    - 'src/operator/nn/im2col.h'
+    - 'src/operator/contrib/nn/deformable_im2col.cuh'
+    - 'src/operator/contrib/nn/deformable_im2col.h'
+    - 'src/operator/contrib/nn/modulated_deformable_im2col.cuh'
+    - 'src/operator/contrib/nn/modulated_deformable_im2col.h'
     # files licensed under bsd 3-clause
     - 'cmake/upstream/FindCUDAToolkit.cmake'
     - 'cmake/upstream/select_compute_arch.cmake'
+    - 'python/mxnet/onnx/mx2onnx/_export_onnx.py'
+    - 'python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset12.py'
+    - 'python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset13.py'
     - 'src/operator/contrib/erfinv-inl.h'
     - 'src/operator/numpy/np_einsum_op-inl.h'
     - 'src/operator/numpy/np_einsum_op.cc'
     - 'src/operator/numpy/np_einsum_path_op-inl.h'
-    # files licensed under caffe/mit license
+    # files licensed under mit license
     - 'src/operator/contrib/modulated_deformable_convolution-inl.h'
     - 'src/operator/contrib/modulated_deformable_convolution.cc'
     - 'src/operator/contrib/modulated_deformable_convolution.cu'
-    - 'src/operator/contrib/nn/deformable_im2col.cuh'
-    - 'src/operator/contrib/nn/deformable_im2col.h'
-    - 'src/operator/contrib/nn/modulated_deformable_im2col.cuh'
-    - 'src/operator/contrib/nn/modulated_deformable_im2col.h'
-    - 'src/operator/nn/im2col.cuh'
-    - 'src/operator/nn/im2col.h'
-    - 'src/operator/nn/pool.cuh'
-    - 'src/operator/nn/pool.h'
+    - 'src/operator/nn/layer_norm_cpu.h'
     # symlinks
     - 'include/dlpack' # symlink to 3rdparty/dlpack/include/dlpack
     - 'include/dmlc' # symlink to 3rdparty/dmlc-core/include/dmlc

LICENSE

@@ -207,8 +207,7 @@
     The Apache MXNET (incubating) project contains subcomponents with separate
     copyright notices and license terms. Your use of the source code for the
     these subcomponents is subject to the terms and conditions of the following
-    licenses. If not stated otherwise, their copyright notices and license terms
-    are available at the path of the subcomponent.
+    licenses. See licenses/ for text of these licenses.
 
     If a folder hierarchy is listed as subcomponent, separate listings of
     further subcomponents (files or folder hierarchies) part of the hierarchy
@@ -220,25 +219,25 @@
 
     3rdparty/ctc_include
     3rdparty/dlpack
-    include/dlpack (header symlinks to 3rdparty/dlpack/include/dlpack)
     3rdparty/dmlc-core
-    include/dmlc (header symlinks to 3rdparty/dmlc-core/include/dmlc)
+    3rdparty/googletest/googlemock/scripts/generator
     3rdparty/mshadow
-    include/mshadow (header symlinks to 3rdparty/mshadow/mshadow)
-    3rdparty/tvm
-    3rdparty/tvm/3rdparty/dmlc-core
-    3rdparty/tvm/3rdparty/dlpack
-    include/nnvm (header symlinks to 3rdparty/tvm/nnvm/include/nnvm)
-    3rdparty/ps-lite
     3rdparty/mkldnn
-    include/mkldnn (header symlinks to 3rdparty/mkldnn)
-    3rdparty/googletest/googlemock/scripts/generator
+    3rdparty/mkldnn/doc/assets/mathjax
+    3rdparty/mkldnn/tests/benchdnn
+    3rdparty/mkldnn/cmake/FindComputeCpp.cmake (see licenses/LICENSE.mkldnn.txt)
     3rdparty/onnx-tensorrt/third_party/onnx/third_party/benchmark
     3rdparty/onnx-tensorrt/third_party/onnx/tools/protoc-gen-mypy.py
-    3rdparty/mkldnn/tests/benchdnn (Copy of the License available at top of current file)
-    src/operator/special_functions-inl.h Cephes Library Functions (Copy of the License available at top of current file)
-    3rdparty/mkldnn/doc/assets/mathjax (Copy of the License available at top of current file)
-    3rdparty/tvm/3rdparty/bfloat16/bfloat16.cc (Copy of the License available at top of current file)
+    3rdparty/ps-lite
+    3rdparty/tvm
+    3rdparty/tvm/3rdparty/dlpack
+    3rdparty/tvm/3rdparty/dmlc-core
+    3rdparty/tvm/3rdparty/bfloat16/bfloat16.cc
+    include/dlpack (header symlinks to 3rdparty/dlpack/include/dlpack)
+    include/dmlc (header symlinks to 3rdparty/dmlc-core/include/dmlc)
+    include/mkldnn (header symlinks to 3rdparty/mkldnn)
+    include/mshadow (header symlinks to 3rdparty/mshadow/mshadow)
+    include/nnvm (header symlinks to 3rdparty/tvm/nnvm/include/nnvm)
     src/operator/contrib/deformable_convolution-inl.h
     src/operator/contrib/deformable_convolution.cc
     src/operator/contrib/deformable_convolution.cu
@@ -251,29 +250,35 @@
     src/operator/contrib/psroi_pooling.cc
     src/operator/contrib/psroi_pooling.cu
     src/operator/nn/mkldnn/mkldnn_base-inl.h
+    src/operator/special_functions-inl.h
 
     =======================================================================================
     MIT license
     =======================================================================================
 
-    example/gluon/tree_lstm
-    3rdparty/tvm/3rdparty/cma
+    3rdparty/intgemm
+    3rdparty/mkldnn/src/gpu/jit/ngen/npack/elf_structs.hpp (see licenses/LICENSE.mkldnn.txt)
+    3rdparty/mkldnn/src/gpu/jit/ngen/npack/hash.hpp (see licenses/LICENSE.mkldnn.txt)
     3rdparty/onnx-tensorrt
     3rdparty/onnx-tensorrt/third_party/onnx
-    3rdparty/intgemm
+    3rdparty/tvm/3rdparty/cma
     3rdparty/tvm/3rdparty/compiler-rt/builtin_fp16.h
+    example/gluon/tree_lstm
     src/operator/contrib/modulated_deformable_convolution-inl.h
     src/operator/contrib/modulated_deformable_convolution.cc
     src/operator/contrib/modulated_deformable_convolution.cu
+    src/operator/nn/layer_norm_cpu.h
 
     =======================================================================================
     3-clause BSD license
     =======================================================================================
 
-    3rdparty/mkldnn/src/cpu/x64/xbyak
-    3rdparty/mkldnn/tests/gtests/gtest
-    3rdparty/mkldnn/cmake/FindOpenCL.cmake (Copy of the License available at licenses/BSD3-cmake)
-    3rdparty/mkldnn/src/cpu/x64/jit_utils/jitprofiling/
+    3rdparty/mkldnn/cmake/FindACL.cmake (see licenses/LICENSE.mkldnn.txt)
+    3rdparty/mkldnn/cmake/FindBLAS.cmake (see licenses/LICENSE.mkldnn.txt)
+    3rdparty/mkldnn/cmake/FindOpenCL.cmake (see licenses/LICENSE.mkldnn.txt)
+    3rdparty/mkldnn/src/cpu/x64/jit_utils/jitprofiling (see licenses/LICENSE.mkldnn.txt)
+    3rdparty/mkldnn/src/cpu/x64/xbyak (see licenses/LICENSE.mkldnn.txt)
+    3rdparty/mkldnn/tests/gtests/gtest (see licenses/LICENSE.mkldnn.txt)
     3rdparty/onnx-tensorrt/third_party/onnx/third_party/pybind11/tools/FindPythonLibsNew.cmake
     3rdparty/ctc_include/contrib/moderngpu
     3rdparty/nvidia_cub
@@ -282,6 +287,9 @@
     3rdparty/googletest/googletest
     cmake/upstream/FindCUDAToolkit.cmake
     cmake/upstream/select_compute_arch.cmake
+    python/mxnet/onnx/mx2onnx/_export_onnx.py
+    python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset12.py
+    python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset13.py
     src/operator/contrib/erfinv-inl.h
     src/operator/numpy/np_einsum_op-inl.h
     src/operator/numpy/np_einsum_path_op-inl.h
@@ -292,11 +300,10 @@
     =======================================================================================
 
     3rdparty/dmlc-core/include/dmlc/concurrentqueue.h
-    include/dmlc/concurrentqueue.h (symlink to 3rdparty/dmlc-core/include/dmlc/concurrentqueue.h)
-    3rdparty/onnx-tensorrt/third_party/onnx/third_party/pybind11/tools/FindEigen3.cmake  (Copy of the License available at licenses/BSD2)
-    3rdparty/onnx-tensorrt/third_party/onnx/third_party/pybind11/tools/FindPythonLibsNew.cmake
+    3rdparty/onnx-tensorrt/third_party/onnx/third_party/pybind11/tools/FindEigen3.cmake
     3rdparty/tvm/3rdparty/picojson/picojson.h
     example/ssd/dataset/pycocotools/coco.py
+    include/dmlc/concurrentqueue.h (symlink to 3rdparty/dmlc-core/include/dmlc/concurrentqueue.h)
 
     =======================================================================================
     Apache-2.0 license + LLVM Exceptions
@@ -305,7 +312,7 @@
     3rdparty/openmp
 
     =======================================================================================
-    Caffe Licensing Model
+    2-clause BSD license + Caffe Copyright Notice and Disclaimer
     =======================================================================================
 
     src/operator/nn/pool.h
@@ -324,26 +331,12 @@
     3rdparty/dmlc-core/include/dmlc/blockingconcurrentqueue.h
     include/dmlc/blockingconcurrentqueue.h (symlink to 3rdparty/dmlc-core/include/dmlc/blockingconcurrentqueue.h)
 
-    =======================================================================================
-    Apache-2.0 license + 3-clause BSD license
-    =======================================================================================
-
-    python/mxnet/onnx/mx2onnx/_export_onnx.py
-    python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset12.py
-    python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset13.py
-
-    =======================================================================================
-    Apache-2.0 license + MIT License
-    =======================================================================================
-
-    src/operator/nn/layer_norm.cc (function LayerNormCPUKernel is adapated from MIT-licensed code)
-
     =======================================================================================
     Boost Software License, Version 1.0
     =======================================================================================
 
-    3rdparty/intgemm/test/3rd_party/catch.hpp  (Copy of the License available at licenses/BOOST1_0)
-    3rdparty/mkldnn/src/common/primitive_hashing.hpp
+    3rdparty/intgemm/test/3rd_party/catch.hpp
+    3rdparty/mkldnn/src/common/primitive_hashing.hpp (see licenses/LICENSE.mkldnn.txt)
     cmake/Modules/FindJeMalloc.cmake
 
     =======================================================================================

ci/build_windows.py

@@ -151,6 +151,8 @@ class BuildFlavour(Enum):
 
 def windows_build(args):
     logging.info("Using vcvars environment:\n{}".format(args.vcvars))
+    if args.vcvars_ver:
+        logging.info("Using vcvars version:\n{}".format(args.vcvars_ver))
 
     path = args.output
 
@@ -171,13 +173,22 @@ def windows_build(args):
             env = os.environ.copy()
             if 'GPU' in args.flavour:
                 env["CXXFLAGS"] = '/FS /MD /O2 /Ob2'
-            cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
-                                                         CMAKE_FLAGS[args.flavour],
-                                                         mxnet_root)
+            if not args.vcvars_ver:
+                cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
+                                                             CMAKE_FLAGS[args.flavour],
+                                                             mxnet_root)
+            else:
+                cmd = "\"{}\" -vcvars_ver={} && cmake -GNinja {} {}".format(args.vcvars,
+                                                                            args.vcvars_ver,
+                                                                            CMAKE_FLAGS[args.flavour],
+                                                                            mxnet_root)
             logging.info("Generating project with CMake:\n{}".format(cmd))
             check_call(cmd, shell=True, env=env)
 
-            cmd = "\"{}\" && ninja".format(args.vcvars)
+            if not args.vcvars_ver:
+                cmd = "\"{}\" && ninja".format(args.vcvars)
+            else:
+                cmd = "\"{}\" -vcvars_ver={} && ninja".format(args.vcvars, args.vcvars_ver)
             logging.info("Building:\n{}".format(cmd))
 
             t0 = int(time.time())
@@ -260,6 +271,12 @@ def main():
         default=KNOWN_VCVARS['VS 2019'],
         type=str)
 
+    parser.add_argument("--vcvars_ver",
+        help="Optionally specifies the Visual Studio compiler toolset to use.\
+            By default, the environment is set to use the current Visual Studio compiler toolset.",
+        default=None,
+        type=str)
+
     parser.add_argument("--arch",
         help="architecture",
         default='x64',

ci/docker/install/requirements

@@ -20,7 +20,7 @@
 
 boto3==1.9.229
 cpplint==1.3.0
-Cython==0.29.7
+Cython==0.29.24
 decorator==4.4.0
 h5py<3
 mock==2.0.0
@@ -32,4 +32,4 @@ astroid==2.3.3  # pylint and astroid need to be aligned
 requests<2.19.0,>=2.18.4
 scipy==1.2.1
 setuptools
-coverage
+coverage

ci/jenkins/Jenkins_steps.groovy

@@ -558,7 +558,7 @@ def compile_windows_cpu() {
         ws('workspace/build-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git_win()
-            powershell 'py -3 ci/build_windows.py -f WIN_CPU'
+            powershell 'py -3 ci/build_windows.py -f WIN_CPU --vcvars_ver 14.28'
             stash includes: 'windows_package.7z', name: 'windows_package_cpu'
           }
         }
@@ -572,7 +572,7 @@ def compile_windows_cpu_mkldnn() {
         ws('workspace/build-cpu-mkldnn') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git_win()
-            powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKLDNN'
+            powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKLDNN --vcvars_ver 14.28'
             stash includes: 'windows_package.7z', name: 'windows_package_cpu_mkldnn'
           }
         }
@@ -586,7 +586,7 @@ def compile_windows_cpu_mkldnn_mkl() {
         ws('workspace/build-cpu-mkldnn-mkl') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git_win()
-            powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKLDNN_MKL'
+            powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKLDNN_MKL --vcvars_ver 14.28'
             stash includes: 'windows_package.7z', name: 'windows_package_cpu_mkldnn_mkl'
           }
         }
@@ -600,7 +600,7 @@ def compile_windows_cpu_mkl() {
         ws('workspace/build-cpu-mkl') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git_win()
-            powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKL'
+            powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKL --vcvars_ver 14.28'
             stash includes: 'windows_package.7z', name: 'windows_package_cpu_mkl'
           }
         }
@@ -614,7 +614,7 @@ def compile_windows_gpu() {
         ws('workspace/build-gpu') {
           timeout(time: max_time, unit: 'MINUTES') {
               utils.init_git_win()
-              powershell 'py -3 ci/build_windows.py -f WIN_GPU'
+              powershell 'py -3 ci/build_windows.py -f WIN_GPU --vcvars_ver 14.28'
               stash includes: 'windows_package.7z', name: 'windows_package_gpu'
           }
         }
@@ -628,7 +628,7 @@ def compile_windows_gpu_mkldnn() {
         ws('workspace/build-gpu') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git_win()
-            powershell 'py -3 ci/build_windows.py -f WIN_GPU_MKLDNN'
+            powershell 'py -3 ci/build_windows.py -f WIN_GPU_MKLDNN --vcvars_ver 14.28'
             stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn'
           }
         }

ci/windows/test_py3_cpu.ps1

@@ -23,12 +23,12 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 $env:MXNET_SUBGRAPH_VERBOSE=0
 $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')
 
-C:\Python37\Scripts\pip install -r tests\requirements.txt
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
+C:\Python38\Scripts\pip install -r tests\requirements.txt
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
 if ($LastExitCode -ne 0) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train
 if ($LastExitCode -ne 0) { Throw ("Error running train tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
 # Adding this extra test since it's not possible to set env var on the fly in Windows.
 $env:MXNET_SAFE_ACCUMULATION=1
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest\test_operator.py:test_norm
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest\test_operator.py:test_norm
 if ($LastExitCode -ne 0) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }

ci/windows/test_py3_gpu.ps1

@@ -23,16 +23,16 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 $env:MXNET_SUBGRAPH_VERBOSE=0
 $env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')
 
-C:\Python37\Scripts\pip install -r tests\requirements.txt
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
+C:\Python38\Scripts\pip install -r tests\requirements.txt
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
 if ($LastExitCode -ne 0) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py
 if ($LastExitCode -ne 0) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py
 if ($LastExitCode -ne 0) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train
 if ($LastExitCode -ne 0) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
 # Adding this extra test since it's not possible to set env var on the fly in Windows.
 $env:MXNET_SAFE_ACCUMULATION=1
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_norm
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_norm
 if ($LastExitCode -ne 0) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }