[WIP] Test CI build (apache#10111)

* Refreshed branch my_cython

* Trigger

* lint

* Set master on mshadow

* Don't need sys/time.h

* don't use REQUIRE for find_package() python, cython

* Add Windows dll linkage to cython test API

* Add python and cython installs

* Set openmp commit

* Added unit test

ci/docker/Dockerfile.build.jetson

@@ -59,6 +59,7 @@ RUN JETPACK_DOWNLOAD_PREFIX=http://developer.download.nvidia.com/devzone/devcent
     dpkg -i $ARM_CUDNN_DEV_INSTALLER_PACKAGE && \
     apt update -y  && \
     apt install -y unzip cuda-cudart-cross-aarch64-8-0 cuda-cublas-cross-aarch64-8-0 \
+    apt install -y python python-dev python3 python3-dev cython cython3 \
     cuda-nvml-cross-aarch64-8-0 cuda-nvrtc-cross-aarch64-8-0 cuda-cufft-cross-aarch64-8-0 \
     cuda-curand-cross-aarch64-8-0 cuda-cusolver-cross-aarch64-8-0 cuda-cusparse-cross-aarch64-8-0 \
     cuda-misc-headers-cross-aarch64-8-0 cuda-npp-cross-aarch64-8-0 libcudnn6  && \

cmake/UseCython.cmake

@@ -96,10 +96,10 @@ if(NOT python_libs_version)
   message(STATUS "Looking for python dependencies, version: ${python_libs_version}")
 endif()
 
-find_package(PythonInterp ${python_libs_version} REQUIRED)
+find_package(PythonInterp ${python_libs_version})
 if(PYTHONINTERP_FOUND)
   message(STATUS "Python ${python_libs_version} executable: ${PYTHON_EXECUTABLE}")
-  find_package(PythonLibs ${python_libs_version} REQUIRED)
+  find_package(PythonLibs ${python_libs_version})
   if(PYTHONLIBS_FOUND)
     set(PYTHON_DEBUG_LIBRARY ${PYTHON_LIBRARY})
     set(PYTHON_DEBUG_LIBRARIES ${PYTHON_DEBUG_LIBRARIES})
@@ -110,7 +110,7 @@ if(PYTHONINTERP_FOUND)
     list(GET PYTHON_VERSION_LIST 1 PYTHON_VERSION_MINOR)
     list(GET PYTHON_VERSION_LIST 1 PYTHON_VERSION_PATCH)
 
-    find_package(Cython ${python_libs_version} REQUIRED)
+    find_package(Cython ${python_libs_version})
     if(CYTHON_FOUND)
       set(CYTHON${python_libs_version}_FOUND ${python_libs_version})
       message(STATUS  " CYTHON${python_libs_version}_FOUND: ${CYTHON${python_libs_version}_FOUND}")

python/mxnet/cython/__init__.py

@@ -14,3 +14,32 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
+import importlib
+import sys
+
+
+def load_cython(package_name, module_name):
+    # How to pass something like '.' as package_name?
+    name = package_name + '.cyX.' + module_name
+    try:
+        if sys.version_info >= (3, 0):
+            if len(package_name) > 0 and package_name[0] != '.':
+                name = package_name + '.cy3.' + module_name
+                package_name = None
+            else:
+                name = 'cy3.' + module_name
+        else:
+            if len(package_name) > 0 and package_name[0] != '.':
+                name = package_name + '.cy2.' + module_name
+                package_name = None
+            else:
+                name = 'cy2.' + module_name
+        #print('Attemptiog to load cython module: {}'.format(name))
+        the_module = importlib.import_module(name, package=package_name)
+        #print('Loaded cython module: {}'.format(name))
+        return the_module
+    except:
+        # No cython found
+        print('Unable to load cython module: {}'.format(name))
+    return None

src/cython/cpp_api.cc

@@ -18,12 +18,11 @@
  */
 #include <iostream>
 #include <cstdarg>
-#include <sys/time.h>
 #include <chrono>
-#include "cpp_api.h"
+#include "./cpp_api.h"
 
 extern "C" int CythonPrintFromCPP(const char *foo) {
-  if(foo) {
+  if (foo) {
     std::cout << foo << std::endl << std::flush;
   }
   return 0;

src/cython/cpp_api.h

@@ -19,23 +19,25 @@
 #ifndef MXNET_CYTHON_CPP_API_H_
 #define MXNET_CYTHON_CPP_API_H_
 
+#include <mxnet/c_api.h>
+
 /*! \brief Inhibit C++ name-mangling for MXNet functions. */
 #ifdef __cplusplus
 extern "C" {
 #endif  // __cplusplus
 
-int CythonPrintFromCPP(const char *foo);
-int Printf(const char *fmt, ...);
-int TrivialCPPCall(int var);
-uint64_t TimeInMilliseconds();
+MXNET_DLL int CythonPrintFromCPP(const char *foo);
+MXNET_DLL int Printf(const char *fmt, ...);
+MXNET_DLL int TrivialCPPCall(int var);
+MXNET_DLL uint64_t TimeInMilliseconds();
 
 #ifdef __cplusplus
 }
 #endif  // __cplusplus
 
 namespace shapes {
 
-class Rectangle {
+class MXNET_DLL Rectangle {
  public:
   int x0, y0, x1, y1;
   Rectangle();

tests/python/unittest/test_cython.py

@@ -20,22 +20,11 @@
 
 from __future__ import print_function
 import sys
-import time
 from mxnet.base import _LIB
+import mxnet.cython as cy
 
-try:
-  if sys.version_info >= (3, 0):
-    import mxnet.cython.cy3.mxcython as mxc
-    import mxnet.ndarray.cy3.ndarray as ndcy
-    import mxnet.symbol.cy3.symbol   as symcy
-  else:
-    import mxnet.cython.cy2.mxcython as mxc
-    import mxnet.ndarray.cy2.ndarray as ndcy
-    import mxnet.symbol.cy2.symbol   as symcy
-except:
-  # No cython found
-  print('Unable to load cython modules')
-  exit(1)
+mxc  = cy.load_cython('mxnet.cython', 'mxcython')
+cynd = cy.load_cython('mxnet.ndarray', 'ndarray')
 
 def test_basic_cython():
   print('ENTER test_basic_cython')
@@ -69,6 +58,7 @@ def test_perf(count, make_c_call):
     msg = " WITH API CALL"
   print("PYTHON {}: {} items took {} seconds".format(msg, count, float(stop - start)/1000))
 
+
 def test_perf_bridge(count, do_cython_call, api_call_count):
   if do_cython_call == 0:
     assert api_call_count == 0  # Sanity on input values