Skip to content

[Android] Move add header files into AAR to using Gradle #8068

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
guoyu-wang merged 2 commits into from
Jun 16, 2021
Merged

[Android] Move add header files into AAR to using Gradle #8068

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fd5644a
Move add header files into AAR to using Gradle
guoyu-wang Jun 16, 2021
a6d0922
fix gradle format violation
guoyu-wang Jun 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmake/onnxruntime_java.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Android")
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime> ${ANDROID_PACKAGE_ABI_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime>)
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime4j_jni> ${ANDROID_PACKAGE_ABI_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime4j_jni>)
# Generate the Android AAR package
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${GRADLE_EXECUTABLE} -b build-android.gradle -c settings-android.gradle build -DjniLibsDir=${ANDROID_PACKAGE_JNILIBS_DIR} -DbuildDir=${ANDROID_PACKAGE_OUTPUT_DIR} -DminSdkVer=${ANDROID_MIN_SDK} WORKING_DIRECTORY ${JAVA_ROOT})
add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${GRADLE_EXECUTABLE} -b build-android.gradle -c settings-android.gradle build -DjniLibsDir=${ANDROID_PACKAGE_JNILIBS_DIR} -DbuildDir=${ANDROID_PACKAGE_OUTPUT_DIR} -DminSdkVer=${ANDROID_MIN_SDK} -DheadersDir=${ANDROID_HEADERS_DIR} WORKING_DIRECTORY ${JAVA_ROOT})

if (onnxruntime_BUILD_UNIT_TESTS)
set(ANDROID_TEST_PACKAGE_ROOT ${JAVA_ROOT}/src/test/android)
Expand Down
30 changes: 30 additions & 0 deletions java/build-android.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ apply plugin: 'maven-publish'

def jniLibsDir = System.properties['jniLibsDir']
def buildDir = System.properties['buildDir']
def headersDir = System.properties['headersDir']
def publishDir = System.properties['publishDir']
def minSdkVer = System.properties['minSdkVer']
def targetSdkVer = System.properties['targetSdkVer']
Expand Down Expand Up @@ -162,3 +163,32 @@ publishing {
}
}
}

// Add ORT C and C++ API headers to the AAR package, after task bundleDebugAar or bundleReleaseAar
// Such that developers using ORT native API can extract libraries and headers from AAR package without building ORT
tasks.whenTaskAdded { task ->
if (task.name.startsWith("bundle") && task.name.endsWith("Aar")) {
doLast {
addFolderToAar("addHeadersTo" + task.name, task.archivePath, headersDir, 'headers')
}
}
}

def addFolderToAar(taskName, aarPath, folderPath, folderPathInAar) {
def tmpDir = file("${buildDir}/${taskName}")
tmpDir.mkdir()
def tmpDirFolder = file("${tmpDir.path}/${folderPathInAar}")
tmpDirFolder.mkdir()
copy {
from zipTree(aarPath)
into tmpDir
}
copy {
from fileTree(folderPath)
into tmpDirFolder
}
ant.zip(destfile: aarPath) {
fileset(dir: tmpDir.path)
}
delete tmpDir
}
117 changes: 45 additions & 72 deletions tools/ci_build/github/android/build_aar_package.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,35 +30,35 @@


def _parse_build_settings(args):
_setting_file = args.build_settings_file.resolve()
setting_file = args.build_settings_file.resolve()

if not _setting_file.is_file():
raise FileNotFoundError('Build config file {} is not a file.'.format(_setting_file))
if not setting_file.is_file():
raise FileNotFoundError('Build config file {} is not a file.'.format(setting_file))

with open(_setting_file) as f:
_build_settings_data = json.load(f)
with open(setting_file) as f:
build_settings_data = json.load(f)

build_settings = {}

if 'build_abis' in _build_settings_data:
build_settings['build_abis'] = _build_settings_data['build_abis']
if 'build_abis' in build_settings_data:
build_settings['build_abis'] = build_settings_data['build_abis']
else:
build_settings['build_abis'] = DEFAULT_BUILD_ABIS

build_params = []
if 'build_params' in _build_settings_data:
build_params += _build_settings_data['build_params']
if 'build_params' in build_settings_data:
build_params += build_settings_data['build_params']
else:
raise ValueError('build_params is required in the build config file')

if 'android_min_sdk_version' in _build_settings_data:
build_settings['android_min_sdk_version'] = _build_settings_data['android_min_sdk_version']
if 'android_min_sdk_version' in build_settings_data:
build_settings['android_min_sdk_version'] = build_settings_data['android_min_sdk_version']
else:
build_settings['android_min_sdk_version'] = DEFAULT_ANDROID_MIN_SDK_VER
build_params += ['--android_api=' + str(build_settings['android_min_sdk_version'])]

if 'android_target_sdk_version' in _build_settings_data:
build_settings['android_target_sdk_version'] = _build_settings_data['android_target_sdk_version']
if 'android_target_sdk_version' in build_settings_data:
build_settings['android_target_sdk_version'] = build_settings_data['android_target_sdk_version']
else:
build_settings['android_target_sdk_version'] = DEFAULT_ANDROID_TARGET_SDK_VER

Expand All @@ -72,107 +72,80 @@ def _parse_build_settings(args):
return build_settings


# Add ORT C and C++ API headers to the AAR package (in fact a zip file)
# Such that developers using ORT native API can extract libraries and header from AAR package without building ORT
# TODO, see if we can use Gradle to add headers to AAR package directly, which is necessary if we want to
# publish the packagee directly using Gradle in the pipeline
def _add_headers_to_aar(aar_file_path, header_files_path):
import shutil
import tempfile
with tempfile.TemporaryDirectory() as temp_dir:
aar_content = os.path.join(temp_dir, 'aar_content')
shutil.unpack_archive(aar_file_path, aar_content, 'zip')

# copy necessary header files
shutil.copytree(header_files_path, os.path.join(aar_content, 'headers'))

# create the zip archive
zip_base_filename = os.path.join(temp_dir, 'aar_with_headers')
zip_filename = zip_base_filename + '.zip'
shutil.make_archive(zip_base_filename, 'zip', root_dir=aar_content)

# overwrite the existing AAR package
shutil.move(zip_filename, aar_file_path)


def _build_aar(args):
build_settings = _parse_build_settings(args)
build_dir = os.path.abspath(args.build_dir)

# Setup temp environment for building
_env = os.environ.copy()
_env['ANDROID_HOME'] = os.path.abspath(args.android_sdk_path)
_env['ANDROID_NDK_HOME'] = os.path.abspath(args.android_ndk_path)
temp_env = os.environ.copy()
temp_env['ANDROID_HOME'] = os.path.abspath(args.android_sdk_path)
temp_env['ANDROID_NDK_HOME'] = os.path.abspath(args.android_ndk_path)

# Temp dirs to hold building results
_intermediates_dir = os.path.join(build_dir, 'intermediates')
_build_config = args.config
_aar_dir = os.path.join(_intermediates_dir, 'aar', _build_config)
_jnilibs_dir = os.path.join(_intermediates_dir, 'jnilibs', _build_config)
_base_build_command = [
sys.executable, BUILD_PY, '--config=' + _build_config
intermediates_dir = os.path.join(build_dir, 'intermediates')
build_config = args.config
aar_dir = os.path.join(intermediates_dir, 'aar', build_config)
jnilibs_dir = os.path.join(intermediates_dir, 'jnilibs', build_config)
base_build_command = [
sys.executable, BUILD_PY, '--config=' + build_config
] + build_settings['build_params']
header_files_path = ''

# Build binary for each ABI, one by one
for abi in build_settings['build_abis']:
_build_dir = os.path.join(_intermediates_dir, abi)
_build_command = _base_build_command + [
abi_build_dir = os.path.join(intermediates_dir, abi)
abi_build_command = base_build_command + [
'--android_abi=' + abi,
'--build_dir=' + _build_dir
'--build_dir=' + abi_build_dir
]

if args.include_ops_by_config is not None:
_build_command += ['--include_ops_by_config=' + args.include_ops_by_config]
abi_build_command += ['--include_ops_by_config=' + args.include_ops_by_config]

subprocess.run(_build_command, env=_env, shell=False, check=True, cwd=REPO_DIR)
subprocess.run(abi_build_command, env=temp_env, shell=False, check=True, cwd=REPO_DIR)

# create symbolic links for libonnxruntime.so and libonnxruntime4j_jni.so
# to jnilibs/[abi] for later compiling the aar package
_jnilibs_abi_dir = os.path.join(_jnilibs_dir, abi)
os.makedirs(_jnilibs_abi_dir, exist_ok=True)
abi_jnilibs_dir = os.path.join(jnilibs_dir, abi)
os.makedirs(abi_jnilibs_dir, exist_ok=True)
for lib_name in ['libonnxruntime.so', 'libonnxruntime4j_jni.so']:
_target_lib_name = os.path.join(_jnilibs_abi_dir, lib_name)
target_lib_name = os.path.join(abi_jnilibs_dir, lib_name)
# If the symbolic already exists, delete it first
# For some reason, os.path.exists will return false for a symbolic link in Linux,
# add double check with os.path.islink
if os.path.exists(_target_lib_name) or os.path.islink(_target_lib_name):
os.remove(_target_lib_name)
os.symlink(os.path.join(_build_dir, _build_config, lib_name), _target_lib_name)
if os.path.exists(target_lib_name) or os.path.islink(target_lib_name):
os.remove(target_lib_name)
os.symlink(os.path.join(abi_build_dir, build_config, lib_name), target_lib_name)

# we only need to define the header files path once
if not header_files_path:
header_files_path = os.path.join(_build_dir, _build_config, 'android', 'headers')
header_files_path = os.path.join(abi_build_dir, build_config, 'android', 'headers')

# The directory to publish final AAR
_aar_publish_dir = os.path.join(build_dir, 'aar_out', _build_config)
os.makedirs(_aar_publish_dir, exist_ok=True)
aar_publish_dir = os.path.join(build_dir, 'aar_out', build_config)
os.makedirs(aar_publish_dir, exist_ok=True)

# get the common gradle command args
_gradle_command = [
gradle_command = [
'gradle',
'--no-daemon',
'-b=build-android.gradle',
'-c=settings-android.gradle',
'-DjniLibsDir=' + _jnilibs_dir,
'-DbuildDir=' + _aar_dir,
'-DpublishDir=' + _aar_publish_dir,
'-DjniLibsDir=' + jnilibs_dir,
'-DbuildDir=' + aar_dir,
'-DheadersDir=' + header_files_path,
'-DpublishDir=' + aar_publish_dir,
'-DminSdkVer=' + str(build_settings['android_min_sdk_version']),
'-DtargetSdkVer=' + str(build_settings['android_target_sdk_version'])
]

# If not using shell on Window, will not be able to find gradle in path
_shell = True if is_windows() else False
use_shell = True if is_windows() else False

# clean, build, and publish to a local directory
subprocess.run(_gradle_command + ['clean'], env=_env, shell=_shell, check=True, cwd=JAVA_ROOT)
subprocess.run(_gradle_command + ['build'], env=_env, shell=_shell, check=True, cwd=JAVA_ROOT)

# add C and C++ API headers to the intermediate aar package
aar_file_path = os.path.join(_aar_dir, 'outputs', 'aar', 'onnxruntime-release.aar')
_add_headers_to_aar(aar_file_path, header_files_path)

subprocess.run(_gradle_command + ['publish'], env=_env, shell=_shell, check=True, cwd=JAVA_ROOT)
subprocess.run(gradle_command + ['clean'], env=temp_env, shell=use_shell, check=True, cwd=JAVA_ROOT)
subprocess.run(gradle_command + ['build'], env=temp_env, shell=use_shell, check=True, cwd=JAVA_ROOT)
subprocess.run(gradle_command + ['publish'], env=temp_env, shell=use_shell, check=True, cwd=JAVA_ROOT)


def parse_args():
Expand Down