Support Windows cross-compiling for ARM(64) in ORT build scripts

BUILD.md

@@ -52,7 +52,7 @@ ONNX Runtime python binding only supports Python 3.x. Please use python 3.5+.
     ```
 5. Run `./build.sh --config RelWithDebInfo --build_wheel` for Linux (or `build.bat --config RelWithDebInfo --build_wheel` for Windows)
 
-The build script runs all unit tests by default.
+The build script runs all unit tests by default (for native builds and skips tests by default for cross-compiled builds).
 
 The complete list of build options can be found by running `./build.sh (or ./build.bat) --help`
 
@@ -197,4 +197,11 @@ Please see [ARM docker file](dockerfiles/Dockerfile.arm32v7). Docker build runs
 By doing this, you could avoid hit the ACR-Tasks build timeout (8 hours) 
 
 ### Cross compiling on Windows
-(TODO)
+#### Using Visual C++ compilers
+1. Download and install Visual C++ compilers and libraries for ARM(64). 
+   If you have Visual Studio installed, please use the Visual Studio Installer (look under the section `Individual components` after choosing to `modify` Visual Studio) to download and install the corresponding ARM(64) compilers and libraries.
+
+2. Use `build.bat` and specify `--arm` or `--arm64` as the build option to start building. Preferably use `Developer Command Prompt for VS` or make sure all the installed cross-compilers are findable from the command prompt being used to build using the PATH environmant variable. 
+
+### Using other compilers
+(TODO)

cmake/CMakeLists.txt

@@ -65,6 +65,7 @@ option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable
 option(onnxruntime_USE_NUPHAR "Build with Nupha" OFF)
 option(onnxruntime_USE_BRAINSLICE "Build with BrainSlice" OFF)
 option(onnxruntime_USE_TRT "Build with TensorRT support" OFF)
+option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF)
 
 set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
 #nsync tests failed on Mac Build
@@ -84,6 +85,10 @@ if(onnxruntime_USE_OPENMP)
   endif()
 endif()
 
+if(onnxruntime_CROSS_COMPILING)
+  set(CMAKE_CROSSCOMPILING ON)  
+endif()
+
 #must after OpenMP settings
 find_package(Threads)
 
@@ -339,8 +344,9 @@ if (WIN32)
         string(APPEND CMAKE_CXX_FLAGS " /W4")
     endif()
 
-    # treat warning as error only on x64 platform. For x86, there are too many warnings to fix.
-    if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE)
+    # treat warning as error only on x64 platform. 
+    # For x86 and cross-compiled ARM64 binaries, there are too many warnings to fix, hence ignore warnings for now
+    if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE AND NOT CMAKE_CROSSCOMPILING)
       # treat warnings as errors
       string(APPEND CMAKE_CXX_FLAGS " /WX")
       foreach(type EXE STATIC SHARED)

tools/ci_build/build.py

@@ -38,7 +38,8 @@ def __init__(self, message):
 def parse_arguments():
     parser = argparse.ArgumentParser(description="ONNXRuntime CI build driver.",
                                      usage='''
-Default behavior is --update --build --test.
+Default behavior is --update --build --test for native architecture builds.
+Default behavior is --update --build for cross-compiled builds.
 
 The Update phase will update git submodules, and run cmake to generate makefiles.
 The Build phase will build all projects.
@@ -64,6 +65,7 @@ def parse_arguments():
     parser.add_argument("--enable_onnx_tests", action='store_true',
                         help='''When running the Test phase, run onnx_test_running against available test data directories.''')
     parser.add_argument("--pb_home", help="Path to protobuf installation")
+    parser.add_argument("--path_to_protoc_exe", help="Path to protoc exe. Will be overridden by {pb_home}/bin/protoc.exe if {pb_home} is set.")
     parser.add_argument("--download_test_data", action="store_true",
                         help='''Downloads test data without running the tests''')
     parser.add_argument("--test_data_url", help="Test data URL.")
@@ -93,6 +95,10 @@ def parse_arguments():
                              "These are just CMake -D options without the leading -D.")
     parser.add_argument("--x86", action='store_true',
                         help="Create x86 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed")
+    parser.add_argument("--arm", action='store_true',
+                        help="Create ARM makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed")
+    parser.add_argument("--arm64", action='store_true',
+                        help="Create ARM64 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed")
     parser.add_argument("--msvc_toolset", help="MSVC toolset to use. e.g. 14.11")
 
     # Arguments needed by CI
@@ -271,7 +277,7 @@ def setup_test_data(build_dir, configs, test_data_url, test_data_checksum, azure
                 log.debug("creating shortcut %s -> %s"  % (src_model_dir, dest_model_dir))
                 run_subprocess(['mklink', '/D', '/J', dest_model_dir, src_model_dir], shell=True)
 
-def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, configs, cmake_extra_defines, args, cmake_extra_args):
+def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args):
     log.info("Generating CMake build tree")
     cmake_dir = os.path.join(source_dir, "cmake")
     # TODO: fix jemalloc build so it does not conflict with onnxruntime shared lib builds. (e.g. onnxuntime_pybind)
@@ -300,6 +306,8 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
                  "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"),
                  "-Donnxruntime_USE_EIGEN_THREADPOOL=" + ("ON" if args.use_eigenthreadpool else "OFF"), 
                  "-Donnxruntime_USE_TRT=" + ("ON" if args.use_trt else "OFF"),
+                  # By default - we currently support only cross compiling for ARM/ARM64 (no native compilation supported through this script)
+                 "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"),
                  ]
     if args.use_brainslice:
         bs_pkg_name = args.brain_slice_package_name.split('.', 1)
@@ -326,6 +334,9 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
     if pb_home:
         cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=" + os.path.join(pb_home,'bin','protoc'), '-Donnxruntime_USE_PREBUILT_PB=ON']
 
+    if not pb_home and path_to_protoc_exe:
+        cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=%s" % path_to_protoc_exe]
+
     cmake_args += ["-D{}".format(define) for define in cmake_extra_defines]
 
     if is_windows():
@@ -518,17 +529,46 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda):
         if is_ubuntu_1604():
             run_subprocess([os.path.join(source_dir, 'rename_manylinux.sh')], cwd=cwd+'/dist')
 
+def build_protoc_for_windows_host(cmake_path, source_dir, build_dir):
+    if not is_windows():
+        raise BuildError('Currently only support building protoc for Windows host while cross-compiling for ARM/ARM64 arch')
+
+    log.info("Building protoc for host to be used in cross-compiled build process")
+    protoc_build_dir = os.path.join(build_dir, 'host_protoc')
+    os.makedirs(protoc_build_dir, exist_ok=True)
+    # Generate step
+    cmd_args = [cmake_path,
+                os.path.join(source_dir, 'cmake\external\protobuf\cmake'),
+                '-G',
+                'Visual Studio 15 2017',
+                '-Dprotobuf_BUILD_TESTS=OFF',
+                '-Dprotobuf_WITH_ZLIB_DEFAULT=OFF',
+                '-Dprotobuf_BUILD_SHARED_LIBS=OFF']
+    run_subprocess(cmd_args, cwd= protoc_build_dir)
+    # Build step
+    cmd_args = [cmake_path,
+                "--build", protoc_build_dir,
+                "--config", "Release",
+                "--target", "protoc"]
+    run_subprocess(cmd_args)
+
+    if not os.path.exists(os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe')):
+        raise BuildError("Couldn't build protoc.exe for host. Failing build.")
+
 def main():
     args = parse_arguments()
 
     cmake_extra_defines = args.cmake_extra_defines if args.cmake_extra_defines else []
 
-    # if there was no explicit argument saying what to do, default to update, build and test.
+    # if there was no explicit argument saying what to do, default to update, build and test (for native builds).
     if (args.update == False and args.clean == False and args.build == False and args.test == False):
-        log.debug("Defaulting to running update, build and test.")
+        log.debug("Defaulting to running update, build [and test for native builds].")
         args.update = True
         args.build = True
-        args.test = True
+        if args.arm or args.arm64:
+            args.test = False
+        else:
+            args.test = True
 
     if args.build_wheel:
         args.enable_pybind = True
@@ -556,7 +596,20 @@ def main():
         cmake_extra_args = []
         if(is_windows()):
           if (args.x86):
-            cmake_extra_args = ['-A','Win32','-T','host=x64','-G', 'Visual Studio 15 2017']
+            cmake_extra_args = ['-A','Win32','-G', 'Visual Studio 15 2017']
+          elif (args.arm or args.arm64):
+            # Cross-compiling for ARM(64) architecture
+            # First build protoc for host to use during cross-compilation
+            build_protoc_for_windows_host(cmake_path, source_dir, build_dir)
+            if args.arm:
+                cmake_extra_args = ['-A', 'ARM']
+            else:
+                cmake_extra_args = ['-A', 'ARM64']
+            cmake_extra_args += ['-G', 'Visual Studio 15 2017']
+            # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any)
+            if args.test:
+                log.info("Cannot test on host build machine for cross-compiled ARM(64) builds. Will skip test running after build.")
+                args.test = False
           else:
             toolset = 'host=x64'
             if (args.msvc_toolset):
@@ -566,6 +619,8 @@ def main():
 
             cmake_extra_args = ['-A','x64','-T', toolset, '-G', 'Visual Studio 15 2017']
         if is_ubuntu_1604():
+            if (args.arm or args.arm64):
+                raise BuildError("Only Windows ARM(64) cross-compiled builds supported currently through this script")
             install_ubuntu_deps(args)
             if not is_docker():
                 install_python_deps()
@@ -579,7 +634,14 @@ def main():
                raise UsageError("The test_data_url and test_data_checksum arguments are required.")
             setup_test_data(build_dir, configs, args.test_data_url, args.test_data_checksum, args.azure_sas_key)
 
-        generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, configs, cmake_extra_defines,
+        path_to_protoc_exe = None
+        if args.path_to_protoc_exe:
+            path_to_protoc_exe = args.path_to_protoc_exe
+        # Need to provide path to protoc.exe built for host to be used in the cross-compiled build process
+        elif args.arm or args.arm64:
+            path_to_protoc_exe = os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe')
+
+        generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, path_to_protoc_exe, configs, cmake_extra_defines,
                             args, cmake_extra_args)
 
     if (args.clean):

tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml

@@ -0,0 +1,44 @@
+jobs:
+- job: Windows_ARM_CrossCompile_CI_Dev
+  variables:
+    buildDirectory: '$(Build.BinariesDirectory)'
+  steps:
+    - template: templates/set-test-data-variables-step.yml
+    - task: UniversalPackages@0
+      displayName: 'Download python'
+      inputs:
+        command: download
+        vstsFeed: '$(System.TeamProject)'
+        vstsFeedPackage: 'miniconda3_win64'
+        vstsPackageVersion: '4.5.11'
+        downloadDirectory: '$(Build.BinariesDirectory)\python'
+    - task: CmdLine@1
+      displayName: 'Run python installer'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\python\installer.exe'
+        arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
+      timeoutInMinutes: 10
+    - task: BatchScript@1
+      displayName: 'setup env'
+      inputs:
+        filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat'
+        modifyEnvironment: true
+        workingFolder: '$(Build.BinariesDirectory)'
+    - task: CmdLine@1
+      displayName: 'Download cmake'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
+    - task: CmdLine@1
+      displayName: 'Generate cmake config and build Debug'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm'
+        workingDirectory: "$(Build.BinariesDirectory)"
+    - task: CmdLine@1
+      displayName: 'Generate cmake config and build Release'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm'
+        workingDirectory: "$(Build.BinariesDirectory)"
+    - template: templates/clean-agent-build-directory-step.yml

tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml

@@ -0,0 +1,44 @@
+jobs:
+- job: Windows_ARM_CrossCompile_CI_Dev
+  variables:
+    buildDirectory: '$(Build.BinariesDirectory)'
+  steps:
+    - template: templates/set-test-data-variables-step.yml
+    - task: UniversalPackages@0
+      displayName: 'Download python'
+      inputs:
+        command: download
+        vstsFeed: '$(System.TeamProject)'
+        vstsFeedPackage: 'miniconda3_win64'
+        vstsPackageVersion: '4.5.11'
+        downloadDirectory: '$(Build.BinariesDirectory)\python'
+    - task: CmdLine@1
+      displayName: 'Run python installer'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\python\installer.exe'
+        arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
+      timeoutInMinutes: 10
+    - task: BatchScript@1
+      displayName: 'setup env'
+      inputs:
+        filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat'
+        modifyEnvironment: true
+        workingFolder: '$(Build.BinariesDirectory)'
+    - task: CmdLine@1
+      displayName: 'Download cmake'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
+    - task: CmdLine@1
+      displayName: 'Generate cmake config and build Debug'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm64'
+        workingDirectory: "$(Build.BinariesDirectory)"
+    - task: CmdLine@1
+      displayName: 'Generate cmake config and build Release'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm64'
+        workingDirectory: "$(Build.BinariesDirectory)"
+    - template: templates/clean-agent-build-directory-step.yml