Filter data (Weights) reorder optimization

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "cmake/external/gsl"]
 	path = cmake/external/gsl
 	url = https://github.com/Microsoft/GSL.git
+[submodule "cmake/external/gemmlowp"]
+	path = cmake/external/gemmlowp
+	url = https://github.com/google/gemmlowp.git

cmake/CMakeLists.txt

@@ -374,29 +374,6 @@ else()
   endif()
 endif()
 
-if (onnxruntime_USE_TVM)
-  if (WIN32 AND MSVC)
-    # wd4100: identifier' : unreferenced formal parameter
-    # wd4244: conversion from 'int' to 'char', possible loss of data
-    # wd4251: class X needs to have dll-interface to be used by clients of class Y
-    # wd4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data
-    # wd4275: non dll-interface class X used as base for dll-interface class Y
-    # wd4389: signed/unsigned mismatch
-    # wd4456: declaration of X hides previous local declaration
-    set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456")
-  else()
-    set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=extra" "-Wno-error=ignored-qualifiers")
-    if(HAS_UNUSED_PARAMETER)
-      list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter")
-    endif()
-    if(HAS_CATCH_VALUE)
-      #TODO: send a PR to TVM and fix it
-      list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value")
-    endif()
-  endif()
-  include(onnxruntime_codegen.cmake)
-endif()
-
 if (onnxruntime_USE_JEMALLOC)
   if (Win32)
     message( FATAL_ERROR "Jemalloc is not supported on Windows." )
@@ -412,11 +389,27 @@ include_directories(
   $<TARGET_PROPERTY:GSL,INTERFACE_INCLUDE_DIRECTORIES>
 )
 
+if (onnxruntime_USE_MKLDNN OR onnxruntime_USE_MKLML)
+  include(mkldnn)
+endif()
+
+if (onnxruntime_USE_MKLML)
+  add_definitions(-DUSE_MKLML=1)
+  # USE_MKML_FOR_BLAS may cause numerical differences in tests so disable by default
+  #add_definitions(-DUSE_MKLML_FOR_BLAS=1)
+  if (WIN32 OR APPLE)
+    list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml)
+  else()
+    list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_intel)
+  endif()
+  list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_mklml)
+  link_directories(${MKLML_LIB_DIR})
+endif()
+
 if (onnxruntime_USE_MKLDNN)
   add_definitions(-DUSE_MKLDNN=1)
-  include(mkldnn)
   list(APPEND onnxruntime_EXTERNAL_LIBRARIES mkldnn)
-  list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES mkldnn)
+  list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_mkldnn)
   link_directories(${MKLDNN_LIB_DIR})
 endif()
 
@@ -442,7 +435,6 @@ if (onnxruntime_USE_CUDA)
   endif ()
   file(TO_CMAKE_PATH ${onnxruntime_CUDNN_HOME} onnxruntime_CUDNN_HOME)
   find_package(CUDA 9.0 REQUIRED)
-  include(cub)
   set(CUDA_LINK_LIBRARIES_KEYWORD PRIVATE)
   if (WIN32)
     link_directories(${onnxruntime_CUDNN_HOME}/lib/x64)
@@ -453,7 +445,6 @@ if (onnxruntime_USE_CUDA)
     set(ONNXRUNTIME_CUDA_LIBRARIES cudnn_static cublas_static culibos)
   endif()
   list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${ONNXRUNTIME_CUDA_LIBRARIES})
-  list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES cub)
 
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_30,code=sm_30") # K series
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_50,code=sm_50") # M series
@@ -465,6 +456,33 @@ if (onnxruntime_USE_CUDA)
   endif()
 endif()
 
+if (onnxruntime_USE_TVM)
+  if (WIN32 AND MSVC)
+    # wd4100: identifier' : unreferenced formal parameter
+    # wd4244: conversion from 'int' to 'char', possible loss of data
+    # wd4251: class X needs to have dll-interface to be used by clients of class Y
+    # wd4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data
+    # wd4275: non dll-interface class X used as base for dll-interface class Y
+    # wd4389: signed/unsigned mismatch
+    # wd4456: declaration of X hides previous local declaration
+    set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456")
+  else()
+    set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=extra" "-Wno-error=ignored-qualifiers")
+    if(HAS_UNUSED_PARAMETER)
+      list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter")
+    endif()
+    if(HAS_CATCH_VALUE)
+      #TODO: send a PR to TVM and fix it
+      list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value")
+    endif()
+  endif()
+  include(onnxruntime_codegen.cmake)
+endif()
+
+if (onnxruntime_ENABLE_MICROSOFT_INTERNAL)
+  add_definitions(-DMICROSOFT_INTERNAL)
+endif()
+
 #names in this var must match the directory names under onnxruntime/core/providers
 set(ONNXRUNTIME_PROVIDER_NAMES cpu)
 

cmake/external/mkldnn.cmake

@@ -1,57 +1,73 @@
 include (ExternalProject)
 
 set(MKLDNN_URL https://github.com/intel/mkl-dnn.git)
-# If MKLDNN_TAG is updated, check if platform.cmake.patch needs to be updated.
-set(MKLDNN_TAG v0.17.1)
-set(MKLDNN_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/src/mkl-dnn/src)
-set(MKLDNN_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/install)
-set(MKLDNN_LIB_DIR ${MKLDNN_INSTALL}/lib)
-set(MKLDNN_INCLUDE_DIR ${MKLDNN_INSTALL}/include)
-
-set(MKLDNN_PATCH_COMMAND1 git apply ${CMAKE_SOURCE_DIR}/patches/mkldnn/platform.cmake.patch)
-# discard prior changes due to patching in mkldnn source to unblock incremental builds.
-set(MKLDNN_PATCH_DISCARD_COMMAND cd ${MKLDNN_SOURCE} && git checkout -- .)
+# If MKLDNN_TAG is updated, check if MKLML_VERSION and platform.cmake.patch need to be updated.
+set(MKLDNN_TAG v0.17.2)
+set(MKLML_VERSION 2019.0.1.20180928)
 
 if(WIN32)
+  set(MKLML_OS_VERSION_STR "win")
+  set(MKLML_FILE_EXTENSION "zip")
   set(MKLDNN_SHARED_LIB mkldnn.dll)
   set(MKLDNN_IMPORT_LIB mkldnn.lib)
   if(onnxruntime_USE_MKLML)
-    set(DOWNLOAD_MKLML ${MKLDNN_SOURCE}/scripts/prepare_mkl.bat)
     set(MKLML_SHARED_LIB mklml.dll)
+    set(MKLML_IMPORT_LIB mklml.lib)
     set(IOMP5MD_SHARED_LIB libiomp5md.dll)
+    set(IOMP5MD_IMPORT_LIB libiomp5md.lib)
   endif()
 else()
+  set(MKLML_FILE_EXTENSION "tgz")
   if (APPLE)
     set(MKLDNN_SHARED_LIB libmkldnn.0.dylib)
+    set(MKLML_OS_VERSION_STR "mac")
   else()
     set(MKLDNN_SHARED_LIB libmkldnn.so.0)
+    set(MKLML_OS_VERSION_STR "lnx")
   endif()
   if(onnxruntime_USE_MKLML)
-    set(DOWNLOAD_MKLML ${MKLDNN_SOURCE}/scripts/prepare_mkl.sh)
     set(MKLML_SHARED_LIB libmklml_intel.so)
     set(IOMP5MD_SHARED_LIB libiomp5.so)
   endif()
 endif()
 
-if(NOT onnxruntime_USE_MKLDNN OR EXISTS ${MKLDNN_SOURCE}/external)
-  set(DOWNLOAD_MKLML "")
+if (onnxruntime_USE_MKLML)
+  set(MKLML_URL https://github.com/intel/mkl-dnn/releases/download/${MKLDNN_TAG}/mklml_${MKLML_OS_VERSION_STR}_${MKLML_VERSION}.${MKLML_FILE_EXTENSION})
+
+  ExternalProject_Add(project_mklml
+    PREFIX mklml
+    URL ${MKLML_URL}
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    UPDATE_COMMAND ""
+    INSTALL_COMMAND ""  )
+
+  set(MKML_DIR ${CMAKE_CURRENT_BINARY_DIR}/mklml/src/project_mklml)
+  set(MKLML_INCLUDE_DIR "${MKML_DIR}/include")
+  set(MKLML_LIB_DIR "${MKML_DIR}/lib")
+  link_directories(${MKLML_LIB_DIR})
 endif()
 
-ExternalProject_Add(project_mkldnn
+if (onnxruntime_USE_MKLDNN)
+  set(MKLDNN_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/src/mkl-dnn/src)
+  set(MKLDNN_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/install)
+  set(MKLDNN_LIB_DIR ${MKLDNN_INSTALL}/lib)
+  set(MKLDNN_INCLUDE_DIR ${MKLDNN_INSTALL}/include)
+
+  set(MKLDNN_PATCH_COMMAND1 git apply ${CMAKE_SOURCE_DIR}/patches/mkldnn/platform.cmake.patch)
+  # discard prior changes due to patching in mkldnn source to unblock incremental builds.
+  set(MKLDNN_PATCH_DISCARD_COMMAND cd ${MKLDNN_SOURCE} && git checkout -- .)
+
+  ExternalProject_Add(project_mkldnn
     PREFIX mkl-dnn
     GIT_REPOSITORY ${MKLDNN_URL}
     GIT_TAG ${MKLDNN_TAG}
-    PATCH_COMMAND ${DOWNLOAD_MKLML} COMMAND ${MKLDNN_PATCH_DISCARD_COMMAND} COMMAND ${MKLDNN_PATCH_COMMAND1}
+    PATCH_COMMAND ${MKLDNN_PATCH_DISCARD_COMMAND} COMMAND ${MKLDNN_PATCH_COMMAND1}
     SOURCE_DIR ${MKLDNN_SOURCE}
-    CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL}
-)
-
-if(WIN32)
-  add_library(mkldnn STATIC IMPORTED)
-  set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_IMPORT_LIB})
-else()
-  add_library(mkldnn SHARED IMPORTED)
-  set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_SHARED_LIB})
+    CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL} -DMKLROOT=${MKML_DIR}
+  )
+  link_directories(${MKLDNN_LIB_DIR})
+  if (onnxruntime_USE_MKLML)
+    add_dependencies(project_mkldnn project_mklml)
+  endif()
 endif()
-add_dependencies(mkldnn project_mkldnn)
-include_directories(${MKLDNN_INCLUDE_DIR})

cmake/onnxruntime_providers.cmake

@@ -33,7 +33,7 @@ source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_contrib_ops_srcs})
 add_library(onnxruntime_providers ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs} ${onnxruntime_contrib_ops_srcs})
 onnxruntime_add_include_to_target(onnxruntime_providers onnx protobuf::libprotobuf)
 target_include_directories(onnxruntime_providers PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS})
-add_dependencies(onnxruntime_providers eigen gsl onnx)
+add_dependencies(onnxruntime_providers eigen gsl onnx ${onnxruntime_EXTERNAL_DEPENDENCIES})
 install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cpu  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
 set_target_properties(onnxruntime_providers PROPERTIES LINKER_LANGUAGE CXX)
 set_target_properties(onnxruntime_providers PROPERTIES FOLDER "ONNXRuntime")
@@ -94,7 +94,7 @@ if (onnxruntime_USE_MKLDNN)
   onnxruntime_add_include_to_target(onnxruntime_providers_mkldnn onnx protobuf::libprotobuf)
   add_dependencies(onnxruntime_providers_mkldnn eigen ${onnxruntime_EXTERNAL_DEPENDENCIES})
   set_target_properties(onnxruntime_providers_mkldnn PROPERTIES FOLDER "ONNXRuntime")
-  target_include_directories(onnxruntime_providers_mkldnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS})
+  target_include_directories(onnxruntime_providers_mkldnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${MKLDNN_INCLUDE_DIR} ${MKLML_INCLUDE_DIR})
   install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/mkldnn  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
   set_target_properties(onnxruntime_providers_mkldnn PROPERTIES LINKER_LANGUAGE CXX)
 endif()

cmake/onnxruntime_python.cmake

@@ -182,7 +182,7 @@ if (onnxruntime_USE_MKLML)
   add_custom_command(
     TARGET onnxruntime_pybind11_state POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy
-        ${MKLDNN_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLDNN_LIB_DIR}/${IOMP5MD_SHARED_LIB}
+        ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB}
         $<TARGET_FILE_DIR:${test_data_target}>/onnxruntime/capi/
   )
 endif()

cmake/onnxruntime_unittests.cmake

@@ -437,7 +437,7 @@ if (onnxruntime_USE_MKLML)
   add_custom_command(
     TARGET ${test_data_target} POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy
-    ${MKLDNN_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLDNN_LIB_DIR}/${IOMP5MD_SHARED_LIB}
+    ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB}
     $<TARGET_FILE_DIR:${test_data_target}>
     )
 endif()

cmake/onnxruntime_util.cmake

@@ -9,7 +9,7 @@ file(GLOB_RECURSE onnxruntime_util_srcs
 source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_util_srcs})
 
 add_library(onnxruntime_util ${onnxruntime_util_srcs})
-target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS})
+target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${MKLDNN_INCLUDE_DIR} ${MKLML_INCLUDE_DIR})
 onnxruntime_add_include_to_target(onnxruntime_util onnx protobuf::libprotobuf)
 set_target_properties(onnxruntime_util PROPERTIES LINKER_LANGUAGE CXX)
 set_target_properties(onnxruntime_util PROPERTIES FOLDER "ONNXRuntime")

csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs

@@ -26,7 +26,6 @@ internal enum ErrorCode
         InvalidGraph = 10,
         ShapeInferenceNotRegistered = 11,
         RequirementNotRegistered = 12,
-        ExecutionProviderDLLNotFound = 13
     }
 
     /// <summary>
@@ -49,7 +48,6 @@ public class OnnxRuntimeException: Exception
             { ErrorCode.InvalidGraph, "InvalidGraph" },
             { ErrorCode.ShapeInferenceNotRegistered, "ShapeInferenceNotRegistered" },
             { ErrorCode.RequirementNotRegistered, "RequirementNotRegistered" },
-            { ErrorCode.ExecutionProviderDLLNotFound, "ExecutionProviderDLLNotFound" }
         };
 
         internal OnnxRuntimeException(ErrorCode errorCode, string message)

csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs

@@ -74,7 +74,7 @@ public InferenceSession(string modelPath, SessionOptions options)
             {
                 if (_nativeHandle != IntPtr.Zero)
                 {
-                    NativeMethods.ReleaseONNXSession(_nativeHandle);
+                    NativeMethods.OrtReleaseSession(_nativeHandle);
                     _nativeHandle = IntPtr.Zero;
                 }
                 throw e;
@@ -178,7 +178,7 @@ internal IReadOnlyCollection<NamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxVa
                 {
                     if (outputValueArray[i] != IntPtr.Zero)
                     {
-                        NativeMethods.ReleaseONNXValue(outputValueArray[i]);
+                        NativeMethods.OrtReleaseValue(outputValueArray[i]);
                     }
                 }
                 throw e;
@@ -188,7 +188,7 @@ internal IReadOnlyCollection<NamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxVa
                 // always unpin the input buffers, and delete the native Onnx value objects
                 for (int i = 0; i < inputs.Count; i++)
                 {
-                    NativeMethods.ReleaseONNXValue(inputTensors[i]); // this should not release the buffer, but should delete the native tensor object
+                    NativeMethods.OrtReleaseValue(inputTensors[i]); // this should not release the buffer, but should delete the native tensor object
                     pinnedBufferHandles[i].Dispose();
                 }
             }
@@ -340,7 +340,7 @@ protected virtual void Dispose(bool disposing)
             // cleanup unmanaged resources
             if (_nativeHandle != IntPtr.Zero)
             {
-                NativeMethods.ReleaseONNXSession(_nativeHandle);
+                NativeMethods.OrtReleaseSession(_nativeHandle);
             }
         }