diff --git a/onnxruntime/test/testdata/required_ops_config.readme.txt b/onnxruntime/test/testdata/required_ops_config.readme.txt index a9dc2f19349c0..8edc9d4012908 100644 --- a/onnxruntime/test/testdata/required_ops_config.readme.txt +++ b/onnxruntime/test/testdata/required_ops_config.readme.txt @@ -7,7 +7,7 @@ required_ops.config: List of all the required operators for .onnx models in the testdata directory. Note that add_opset_314159.onnx adds a spurious opset for 'Add' so the config file will contain an entry for that. -required_operators_and_types.config +required_ops_and_types.config To generate/update: python ../../../tools/python/create_reduced_build_config.py --format ORT --enable_type_reduction . required_ops_and_types.config Contains: diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml index 47a9baabed0ce..3ea06591e4daa 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml @@ -7,6 +7,7 @@ # and the models from /onnxruntime/test/testdata/, run UT, and use onnx_test_runner to # test the ort format models generated in step 1. # Exceptions are enabled in this step to help debugging in case of CI failure. +# This step builds and tests ORT with and without type reduction enabled. # 4. Build baseline minimal ORT for Android arm64-v8a including no kernels and disable exceptions # This step is to report the baseline binary size for Android jobs: @@ -15,11 +16,23 @@ jobs: workspace: clean: all pool: Linux-CPU + + variables: + test_data_directory: $(Build.SourcesDirectory)/.test_data + steps: - checkout: self clean: true submodules: recursive + - task: CmdLine@2 + displayName: Create test data directory + inputs: + script: | + # Create a folder for all test data + mkdir -p $(test_data_directory) + workingDirectory: $(Build.SourcesDirectory) + - template: templates/get-docker-image-steps.yml parameters: Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos @@ -31,23 +44,22 @@ jobs: displayName: Build full onnxruntime and generate ORT format test files inputs: script: | - # Create a folder for all test data - mkdir -p $HOME/.test_data docker run --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \ + --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ onnxruntimecentoscpubuild \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 - displayName: Build minimal onnxruntime with exceptions disabled + displayName: Build minimal onnxruntime [exceptions DISABLED, type reduction DISABLED] inputs: script: | - # We will try to build the ORT minimal with exception disabled + # We will try to build minimal ORT with exceptions disabled # Only the building process is verified here, no test will be performed docker run --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ @@ -66,31 +78,59 @@ jobs: --minimal_build \ --disable_exceptions workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 - displayName: Build minimal onnxruntime and run tests with exceptions enabled + displayName: Build minimal onnxruntime [exceptions ENABLED, type reduction DISABLED] and run tests inputs: script: | docker run --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \ + --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ onnxruntimecentoscpubuild \ - /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \ + --build-directory /build/without_type_reduction \ + --reduced-ops-config /home/onnxruntimedev/.test_data/required_ops.ort_models.config workingDirectory: $(Build.SourcesDirectory) + + - script: git checkout -- . + displayName: Discard local changes to Git repository files + workingDirectory: $(Build.SourcesDirectory) + + - task: CmdLine@2 + displayName: Build minimal onnxruntime [exceptions ENABLED, type reduction ENABLED] and run tests + inputs: + script: | + docker run --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimecentoscpubuild \ + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \ + --build-directory /build/with_type_reduction \ + --reduced-ops-config /home/onnxruntimedev/.test_data/required_ops_and_types.ort_models.config \ + --enable-type-reduction + workingDirectory: $(Build.SourcesDirectory) + + - script: git checkout -- . + displayName: Discard local changes to Git repository files + workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 displayName: Build onnxruntime minimal baseline for Android arm64-v8a and report binary size inputs: script: | - # Create a folder for all test data - mkdir -p $HOME/.test_data docker run --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $ANDROID_HOME:/android_home \ - --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \ + --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ diff --git a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh index 8e50be3237ec4..0be9f04d98a27 100644 --- a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh @@ -6,7 +6,7 @@ set -e set -x -# Validate the operator kernel registrations, as the ORT model uses hashes of the kernel registration details +# Validate the operator kernel registrations, as the ORT model uses hashes of the kernel registration details # to find kernels. If the hashes from the registration details are incorrect we will produce a model that will break # when the registration is fixed in the future. python3 /onnxruntime_src/tools/ci_build/op_registration_validator.py @@ -26,25 +26,23 @@ python3 /onnxruntime_src/tools/ci_build/build.py \ # Install the ORT python wheel python3 -m pip install --user /build/Debug/dist/* -# Convert all the E2E ONNX models to ORT format +# Convert all the E2E ONNX models to ORT format python3 /onnxruntime_src/tools/python/convert_onnx_models_to_ort.py \ /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_e2e_test_data -# Create a config with just the required ops for ORT format models in testdata -# This is used by build_minimal_ort_and_run_tests.sh later in the linux-cpu-minimal-build-ci-pipeline CI +# Create configs with just the required ops for ORT format models in testdata +# These are used by build_minimal_ort_and_run_tests.sh later in the linux-cpu-minimal-build-ci-pipeline CI # and will include ops for the E2E models we just converted + +# Config without type reduction python3 /onnxruntime_src/tools/python/create_reduced_build_config.py --format ORT \ /onnxruntime_src/onnxruntime/test/testdata \ - /onnxruntime_src/onnxruntime/test/testdata/required_ops.ort_models.config + /home/onnxruntimedev/.test_data/required_ops.ort_models.config -# Re-create testdata/required_ops_and_types.config. -# This is meaningful when nnapi_minimal_build_minimal_ort_and_run_tests.sh runs later in the -# linux-cpu-minimal-build-ci-pipeline CI, as recreating the configs checks that we are still creating -# a valid config. We have a checked in version of the file for use in other CIs where we don't do a full ORT -# build as part of the CI, such as the android-x86_64-crosscompile-ci-pipeline CI. +# Config with type reduction python3 /onnxruntime_src/tools/python/create_reduced_build_config.py --format ORT --enable_type_reduction \ /onnxruntime_src/onnxruntime/test/testdata \ - /onnxruntime_src/onnxruntime/test/testdata/required_ops_and_types.config + /home/onnxruntimedev/.test_data/required_ops_and_types.ort_models.config # Clear the build rm -rf /build/Debug diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh index 2b70d171ab34f..0947ae12ae282 100644 --- a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh @@ -2,23 +2,75 @@ # This script will create a minimal build with the required operators for all ORT format models # in the testdata directory. This includes E2E models generated by build_full_ort_and_create_ort_files.sh. -# The build will run the unit tests for the minimal build, followed by running onnx_test_runner +# The build will run the unit tests for the minimal build, followed by running onnx_test_runner # for the E2E test cases. - set -e set -x +USAGE_TEXT="Usage: + -b|--build-directory + Specifies the build directory. Required. + -c|--reduced-ops-config + Specifies the reduced Ops configuration file path. Required. + [--enable-type-reduction] + Builds with type reduction enabled." + +BUILD_DIR= +REDUCED_OPS_CONFIG_FILE= +ENABLE_TYPE_REDUCTION= + +while [[ $# -gt 0 ]] +do + OPTION_KEY="$1" + case $OPTION_KEY in + -b|--build-directory) + BUILD_DIR="$2" + shift + shift + ;; + -c|--reduced-ops-config) + REDUCED_OPS_CONFIG_FILE="$2" + shift + shift + ;; + --enable-type-reduction) + ENABLE_TYPE_REDUCTION=1 + shift + ;; + *) + echo "Invalid option: $1" + echo "$USAGE_TEXT" + exit 1 + ;; + esac +done + +if [[ -z "${BUILD_DIR}" || -z "${REDUCED_OPS_CONFIG_FILE}" ]]; then + echo "Required option was not provided." + echo "$USAGE_TEXT" + exit 1 +fi + # Perform a minimal build with required ops and run ORT minimal build UTs python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --cmake_generator Ninja \ + --build_dir ${BUILD_DIR} --cmake_generator Ninja \ --config Debug \ --skip_submodule_sync \ --build_shared_lib \ --parallel \ --minimal_build \ --disable_ml_ops \ - --include_ops_by_config /onnxruntime_src/onnxruntime/test/testdata/required_ops.ort_models.config + --include_ops_by_config ${REDUCED_OPS_CONFIG_FILE} \ + ${ENABLE_TYPE_REDUCTION:+"--enable_reduced_operator_type_support"} # Run the e2e test cases -/build/Debug/onnx_test_runner /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_e2e_test_data +${BUILD_DIR}/Debug/onnx_test_runner /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_e2e_test_data + +# Print binary size info +python3 /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py \ + --arch "$(uname -m)" --os "$(uname -o)" --build_config "minimal-reduced" \ + ${BUILD_DIR}/Debug/libonnxruntime.so + +echo "Binary size info:" +cat ${BUILD_DIR}/Debug/binary_size_data.txt diff --git a/tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py b/tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py index 4fa40172fecd1..e430eb51c898c 100644 --- a/tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py +++ b/tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py @@ -33,7 +33,7 @@ def _check_binary_size(path, readelf, threshold, os_str, arch, build_config): '{},{},{},{}\n'.format(os_str, arch, build_config, sections_total) ]) - if sections_total > threshold: + if threshold is not None and sections_total > threshold: raise RuntimeError('Sections total size for {} of {} exceeds threshold of {} by {}. On-disk size={}' .format(path, sections_total, threshold, sections_total - threshold, ondisk_size)) @@ -42,10 +42,9 @@ def main(): argparser = argparse.ArgumentParser(description='Check the binary size for provided path and ' 'create a text file for upload to the performance dashboard.') - # required - argparser.add_argument('-t', '--threshold', type=int, required=True, - help='Return error if binary size exceeds this threshold.') # optional + argparser.add_argument('-t', '--threshold', type=int, + help='Return error if binary size exceeds this threshold.') argparser.add_argument('-r', '--readelf_path', type=str, default='readelf', help='Path to readelf executable.') argparser.add_argument('--os', type=str, default='android', help='OS value to include in binary_size_data.txt')