Skip to content

Support Windows cross-compiling for ARM(64) in ORT build scripts #549

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hariharans29 merged 14 commits into from
Mar 9, 2019
Merged

Support Windows cross-compiling for ARM(64) in ORT build scripts #549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
ef0dde9
Initial commit
hariharans29 Feb 28, 2019
98f6b05
More changes
hariharans29 Feb 28, 2019
3565efc
More changes
hariharans29 Mar 6, 2019
39ede29
More changes
hariharans29 Mar 6, 2019
a0bfc28
More changes
hariharans29 Mar 6, 2019
bee88ab
Merge branch 'master' into arm64WindowsBuild
hariharans29 Mar 6, 2019
69a45d5
PR feedback
hariharans29 Mar 7, 2019
c04f076
Commiting Azure build config file
hariharans29 Mar 7, 2019
9cd01a3
Fix build pipeline
hariharans29 Mar 7, 2019
1f50d09
Cleanup build dir template addition
hariharans29 Mar 7, 2019
d37c4cb
Remove conda modules download step
hariharans29 Mar 8, 2019
16217e0
PR feedback
hariharans29 Mar 8, 2019
e839792
Revert x86 arguments to as they are currently
hariharans29 Mar 8, 2019
4747e28
More changes
hariharans29 Mar 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions BUILD.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ ONNX Runtime python binding only supports Python 3.x. Please use python 3.5+.
```
5. Run `./build.sh --config RelWithDebInfo --build_wheel` for Linux (or `build.bat --config RelWithDebInfo --build_wheel` for Windows)

The build script runs all unit tests by default.
The build script runs all unit tests by default (for native builds and skips tests by default for cross-compiled builds).

The complete list of build options can be found by running `./build.sh (or ./build.bat) --help`

Expand Down Expand Up @@ -197,4 +197,11 @@ Please see [ARM docker file](dockerfiles/Dockerfile.arm32v7). Docker build runs
By doing this, you could avoid hit the ACR-Tasks build timeout (8 hours)

### Cross compiling on Windows
(TODO)
#### Using Visual C++ compilers
1. Download and install Visual C++ compilers and libraries for ARM(64).
If you have Visual Studio installed, please use the Visual Studio Installer (look under the section `Individual components` after choosing to `modify` Visual Studio) to download and install the corresponding ARM(64) compilers and libraries.

2. Use `build.bat` and specify `--arm` or `--arm64` as the build option to start building. Preferably use `Developer Command Prompt for VS` or make sure all the installed cross-compilers are findable from the command prompt being used to build using the PATH environmant variable.

### Using other compilers
(TODO)
10 changes: 8 additions & 2 deletions cmake/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable
option(onnxruntime_USE_NUPHAR "Build with Nupha" OFF)
option(onnxruntime_USE_BRAINSLICE "Build with BrainSlice" OFF)
option(onnxruntime_USE_TRT "Build with TensorRT support" OFF)
option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF)

set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
#nsync tests failed on Mac Build
Expand All @@ -84,6 +85,10 @@ if(onnxruntime_USE_OPENMP)
endif()
endif()

if(onnxruntime_CROSS_COMPILING)
set(CMAKE_CROSSCOMPILING ON)
endif()

#must after OpenMP settings
find_package(Threads)

Expand Down Expand Up @@ -339,8 +344,9 @@ if (WIN32)
string(APPEND CMAKE_CXX_FLAGS " /W4")
endif()

# treat warning as error only on x64 platform. For x86, there are too many warnings to fix.
if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE)
# treat warning as error only on x64 platform.
# For x86 and cross-compiled ARM64 binaries, there are too many warnings to fix, hence ignore warnings for now
if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE AND NOT CMAKE_CROSSCOMPILING)
# treat warnings as errors
string(APPEND CMAKE_CXX_FLAGS " /WX")
foreach(type EXE STATIC SHARED)
Expand Down
76 changes: 70 additions & 6 deletions tools/ci_build/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ def __init__(self, message):
def parse_arguments():
parser = argparse.ArgumentParser(description="ONNXRuntime CI build driver.",
usage='''
Default behavior is --update --build --test.
Default behavior is --update --build --test for native architecture builds.
Default behavior is --update --build for cross-compiled builds.

The Update phase will update git submodules, and run cmake to generate makefiles.
The Build phase will build all projects.
Expand All @@ -64,6 +65,7 @@ def parse_arguments():
parser.add_argument("--enable_onnx_tests", action='store_true',
help='''When running the Test phase, run onnx_test_running against available test data directories.''')
parser.add_argument("--pb_home", help="Path to protobuf installation")
parser.add_argument("--path_to_protoc_exe", help="Path to protoc exe. Will be overridden by {pb_home}/bin/protoc.exe if {pb_home} is set.")
parser.add_argument("--download_test_data", action="store_true",
help='''Downloads test data without running the tests''')
parser.add_argument("--test_data_url", help="Test data URL.")
Expand Down Expand Up @@ -93,6 +95,10 @@ def parse_arguments():
"These are just CMake -D options without the leading -D.")
parser.add_argument("--x86", action='store_true',
help="Create x86 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed")
parser.add_argument("--arm", action='store_true',
help="Create ARM makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed")
parser.add_argument("--arm64", action='store_true',
help="Create ARM64 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed")
parser.add_argument("--msvc_toolset", help="MSVC toolset to use. e.g. 14.11")

# Arguments needed by CI
Expand Down Expand Up @@ -271,7 +277,7 @@ def setup_test_data(build_dir, configs, test_data_url, test_data_checksum, azure
log.debug("creating shortcut %s -> %s" % (src_model_dir, dest_model_dir))
run_subprocess(['mklink', '/D', '/J', dest_model_dir, src_model_dir], shell=True)

def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, configs, cmake_extra_defines, args, cmake_extra_args):
def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args):
log.info("Generating CMake build tree")
cmake_dir = os.path.join(source_dir, "cmake")
# TODO: fix jemalloc build so it does not conflict with onnxruntime shared lib builds. (e.g. onnxuntime_pybind)
Expand Down Expand Up @@ -300,6 +306,8 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
"-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"),
"-Donnxruntime_USE_EIGEN_THREADPOOL=" + ("ON" if args.use_eigenthreadpool else "OFF"),
"-Donnxruntime_USE_TRT=" + ("ON" if args.use_trt else "OFF"),
# By default - we currently support only cross compiling for ARM/ARM64 (no native compilation supported through this script)
"-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"),
]
if args.use_brainslice:
bs_pkg_name = args.brain_slice_package_name.split('.', 1)
Expand All @@ -326,6 +334,9 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
if pb_home:
cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=" + os.path.join(pb_home,'bin','protoc'), '-Donnxruntime_USE_PREBUILT_PB=ON']

elif path_to_protoc_exe:
cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=%s" % path_to_protoc_exe]

cmake_args += ["-D{}".format(define) for define in cmake_extra_defines]

if is_windows():
Expand Down Expand Up @@ -518,17 +529,48 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda):
if is_ubuntu_1604():
run_subprocess([os.path.join(source_dir, 'rename_manylinux.sh')], cwd=cwd+'/dist')

def build_protoc_for_windows_host(cmake_path, source_dir, build_dir):
if not is_windows():
raise BuildError('Currently only support building protoc for Windows host while cross-compiling for ARM/ARM64 arch')

log.info("Building protoc for host to be used in cross-compiled build process")
protoc_build_dir = os.path.join(build_dir, 'host_protoc')
os.makedirs(protoc_build_dir, exist_ok=True)
# Generate step
cmd_args = [cmake_path,
os.path.join(source_dir, 'cmake\external\protobuf\cmake'),
'-T',
'host=x64',
'-G',
'Visual Studio 15 2017',
'-Dprotobuf_BUILD_TESTS=OFF',
'-Dprotobuf_WITH_ZLIB_DEFAULT=OFF',
'-Dprotobuf_BUILD_SHARED_LIBS=OFF']
run_subprocess(cmd_args, cwd= protoc_build_dir)
# Build step
cmd_args = [cmake_path,
"--build", protoc_build_dir,
"--config", "Release",
"--target", "protoc"]
run_subprocess(cmd_args)

if not os.path.exists(os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe')):
raise BuildError("Couldn't build protoc.exe for host. Failing build.")

def main():
args = parse_arguments()

cmake_extra_defines = args.cmake_extra_defines if args.cmake_extra_defines else []

# if there was no explicit argument saying what to do, default to update, build and test.
# if there was no explicit argument saying what to do, default to update, build and test (for native builds).
if (args.update == False and args.clean == False and args.build == False and args.test == False):
log.debug("Defaulting to running update, build and test.")
log.debug("Defaulting to running update, build [and test for native builds].")
args.update = True
args.build = True
args.test = True
if args.arm or args.arm64:
args.test = False
else:
args.test = True

if args.build_wheel:
args.enable_pybind = True
Expand Down Expand Up @@ -557,6 +599,19 @@ def main():
if(is_windows()):
if (args.x86):
cmake_extra_args = ['-A','Win32','-T','host=x64','-G', 'Visual Studio 15 2017']
elif (args.arm or args.arm64):
# Cross-compiling for ARM(64) architecture
# First build protoc for host to use during cross-compilation
build_protoc_for_windows_host(cmake_path, source_dir, build_dir)
if args.arm:
cmake_extra_args = ['-A', 'ARM']
else:
cmake_extra_args = ['-A', 'ARM64']
cmake_extra_args += ['-G', 'Visual Studio 15 2017']
# Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any)
if args.test:
log.info("Cannot test on host build machine for cross-compiled ARM(64) builds. Will skip test running after build.")
args.test = False
else:
toolset = 'host=x64'
if (args.msvc_toolset):
Expand All @@ -566,6 +621,8 @@ def main():

cmake_extra_args = ['-A','x64','-T', toolset, '-G', 'Visual Studio 15 2017']
if is_ubuntu_1604():
if (args.arm or args.arm64):
raise BuildError("Only Windows ARM(64) cross-compiled builds supported currently through this script")
install_ubuntu_deps(args)
if not is_docker():
install_python_deps()
Expand All @@ -579,7 +636,14 @@ def main():
raise UsageError("The test_data_url and test_data_checksum arguments are required.")
setup_test_data(build_dir, configs, args.test_data_url, args.test_data_checksum, args.azure_sas_key)

generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, configs, cmake_extra_defines,
path_to_protoc_exe = None
if args.path_to_protoc_exe:
path_to_protoc_exe = args.path_to_protoc_exe
# Need to provide path to protoc.exe built for host to be used in the cross-compiled build process
elif args.arm or args.arm64:
path_to_protoc_exe = os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe')

generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, path_to_protoc_exe, configs, cmake_extra_defines,
args, cmake_extra_args)

if (args.clean):
Expand Down
47 changes: 47 additions & 0 deletions tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
jobs:
- job: Windows_ARM_CrossCompile_CI_Dev
variables:
buildDirectory: '$(Build.BinariesDirectory)'
steps:
- template: templates/set-test-data-variables-step.yml
- task: UniversalPackages@0
displayName: 'Download python'
inputs:
command: download
vstsFeed: '$(System.TeamProject)'
vstsFeedPackage: 'miniconda3_win64'
vstsPackageVersion: '4.5.11'
downloadDirectory: '$(Build.BinariesDirectory)\python'
- task: CmdLine@1
displayName: 'Run python installer'
inputs:
filename: '$(Build.BinariesDirectory)\python\installer.exe'
arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
timeoutInMinutes: 10
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- task: CmdLine@1
displayName: 'Download cmake'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
- task: CmdLine@1
displayName: 'Generate cmake config and build Debug'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm'
workingDirectory: "$(Build.BinariesDirectory)"
- task: CmdLine@1
displayName: 'Generate cmake config and build Release'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm'
workingDirectory: "$(Build.BinariesDirectory)"
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: 'Component Detection'
condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
- template: templates/clean-agent-build-directory-step.yml
47 changes: 47 additions & 0 deletions tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
jobs:
- job: Windows_ARM_CrossCompile_CI_Dev
variables:
buildDirectory: '$(Build.BinariesDirectory)'
steps:
- template: templates/set-test-data-variables-step.yml
- task: UniversalPackages@0
displayName: 'Download python'
inputs:
command: download
vstsFeed: '$(System.TeamProject)'
vstsFeedPackage: 'miniconda3_win64'
vstsPackageVersion: '4.5.11'
downloadDirectory: '$(Build.BinariesDirectory)\python'
- task: CmdLine@1
displayName: 'Run python installer'
inputs:
filename: '$(Build.BinariesDirectory)\python\installer.exe'
arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
timeoutInMinutes: 10
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- task: CmdLine@1
displayName: 'Download cmake'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
- task: CmdLine@1
displayName: 'Generate cmake config and build Debug'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm64'
workingDirectory: "$(Build.BinariesDirectory)"
- task: CmdLine@1
displayName: 'Generate cmake config and build Release'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm64'
workingDirectory: "$(Build.BinariesDirectory)"
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: 'Component Detection'
condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
- template: templates/clean-agent-build-directory-step.yml