diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 2f6b5b87c4431..4815e46aab469 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -38,7 +38,7 @@ jobs: use_coreml: true matrix_include: >- [ - {"machine": "arm64", "target": "x86_64", "build_config": "Release"}, + {"machine": "x86_64", "target": "x86_64", "build_config": "Release"}, {"machine": "arm64", "target": "arm64", "build_config": "Debug"}, {"machine": "arm64", "target": "arm64", "build_config": "Release"} ] @@ -61,7 +61,7 @@ jobs: use_webgpu: true matrix_include: >- [ - {"machine": "arm64", "target": "x86_64", "build_config": "Release"}, + {"machine": "x86_64", "target": "x86_64", "build_config": "Release"}, {"machine": "arm64", "target": "arm64", "build_config": "Debug"}, {"machine": "arm64", "target": "arm64", "build_config": "Release"} ] diff --git a/.github/workflows/macos-ci-build-and-test-workflow.yml b/.github/workflows/macos-ci-build-and-test-workflow.yml index 9e435b8a85db8..52fbb16a01909 100644 --- a/.github/workflows/macos-ci-build-and-test-workflow.yml +++ b/.github/workflows/macos-ci-build-and-test-workflow.yml @@ -48,7 +48,7 @@ jobs: # include: ${{ fromJSON(inputs.matrix_include) }} - # "macos-15" is an arm64 image. + # "macos-15-intel" is a x86_64 image, and "macos-15" is an arm64 image. # see also: https://github.com/actions/runner-images/blob/main/README.md runs-on: ${{ matrix.machine == 'x86_64' && 'macos-15-intel' || 'macos-15' }} env: diff --git a/tools/ci_build/github/azure-pipelines/stages/nodejs-npm-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nodejs-npm-packaging-stage.yml index 59424c83a113f..33bae301e9557 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nodejs-npm-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nodejs-npm-packaging-stage.yml @@ -73,6 +73,7 @@ stages: # - Windows arm64 (CPU, DML, WebGPU) # - Linux x64 (CPU, CUDA, TensorRT, WebGPU) # - Linux arm64 (CPU only) + # - macOS x64 (CPU, CoreML, WebGPU) # - macOS arm64 (CPU, CoreML, WebGPU) # # File manifest: @@ -109,6 +110,12 @@ stages: # - onnxruntime_binding.node # - libonnxruntime.so.1 # + # - macOS x64 (CPU, CoreML, WebGPU): + # dependency: MacOS_C_API_Packaging_CPU_x86_64 (drop-onnxruntime-nodejs-osx-x86_64) + # files: + # - onnxruntime_binding.node + # - libonnxruntime.{version}.dylib + # # - macOS arm64 (CPU, CoreML, WebGPU): # stage: MacOS_C_API_Packaging_CPU, job: MacOS_C_API_Packaging_CPU_arm64 (drop-onnxruntime-nodejs-osx-arm64) # files: @@ -136,6 +143,12 @@ stages: artifactName: 'drop-onnxruntime-nodejs-win-arm64' targetPath: '$(Build.BinariesDirectory)/nodejs-artifacts/win32/arm64/' + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Nodejs (macOS x86_64)' + inputs: + artifactName: 'drop-onnxruntime-nodejs-osx-x86_64' + targetPath: '$(Build.BinariesDirectory)/nodejs-artifacts/darwin/x64/' + - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - Nodejs (macOS arm64)' inputs: @@ -212,6 +225,16 @@ stages: *.node TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v6\linux\arm64' + # Node.js binding darwin/x64 + - task: CopyFiles@2 + displayName: 'Copy nodejs binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v6\darwin\x64\' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\nodejs-artifacts\darwin\x64' + Contents: | + libonnxruntime.*.dylib + *.node + TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v6\darwin\x64' + # Node.js binding darwin/arm64 - task: CopyFiles@2 displayName: 'Copy nodejs binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v6\darwin\arm64\' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 10e5aa34727ed..83ee026eb9a0e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -278,6 +278,10 @@ stages: artifactName: drop-onnxruntime-java-linux-aarch64 targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-aarch64' + - input: pipelineArtifact + artifactName: drop-onnxruntime-java-osx-x86_64 + targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x86_64' + - input: pipelineArtifact artifactName: drop-onnxruntime-java-osx-arm64 targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-arm64' diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml index 0bc0a94fdd6e3..af67a5d3ebe0c 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml @@ -26,6 +26,13 @@ stages: jobs: - template: mac-cpu-packing-jobs.yml parameters: + MacosArch: 'x86_64' + AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} + AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} + + - template: mac-cpu-packing-jobs.yml + parameters: + MacosArch: 'arm64' AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} @@ -40,6 +47,9 @@ stages: - ImageOverride -equals ACES_VM_SharedPool_Sequoia templateContext: inputs: + - input: pipelineArtifact + artifactName: onnxruntime-osx-x86_64 + targetPath: $(Build.ArtifactStagingDirectory) - input: pipelineArtifact artifactName: onnxruntime-osx-arm64 # The files in this artifact are not signed targetPath: $(Build.ArtifactStagingDirectory) @@ -56,16 +66,12 @@ stages: architecture: arm64 addToPath: true - - script: | - set -ex - cd $(Build.ArtifactStagingDirectory) - # Find and extract the arm64 tarball - find . -name 'onnxruntime-osx-arm64*.tgz' -exec tar -xzf {} \; - # Remove _manifest directories if they exist - find . -type d -name '_manifest' -exec rm -rf {} + || true - # Find the extracted directory and zip it - find . -maxdepth 1 -type d -name 'onnxruntime-osx-arm64*' -exec zip -FSr --symlinks {}.zip {} \; - displayName: 'Prepare ARM64 Package for Signing' + - task: PythonScript@0 + displayName: 'Prepare, Create Universal Binary, and Zip with Python' + inputs: + scriptSource: 'filePath' + scriptPath: 'tools/ci_build/prepare_macos_package.py' + arguments: '--staging_dir $(Build.ArtifactStagingDirectory)' - template: mac-esrp-dylib.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml index ea196cddafa7d..0941b7192d95c 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml @@ -1,7 +1,9 @@ parameters: - name: MacosArch type: string - default: 'arm64' + values: + - 'x86_64' + - 'arm64' - name: AdditionalBuildFlags displayName: Additional build flags for build.py @@ -59,6 +61,9 @@ steps: - template: nodejs-artifacts-package-and-publish-steps-posix.yml parameters: - arch: arm64 + ${{ if eq(parameters.MacosArch, 'x86_64') }}: + arch: x64 + ${{ if eq(parameters.MacosArch, 'arm64') }}: + arch: arm64 os: 'darwin' artifactName: 'drop-onnxruntime-nodejs-osx-${{ parameters.MacosArch }}' diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml index 0928a19559cd5..370c872b9101d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml @@ -3,6 +3,11 @@ parameters: displayName: Additional build flags for build.py type: string default: '' +- name: MacosArch + type: string + values: + - 'x86_64' + - 'arm64' # Must be 1 or 0 - name: AllowReleasedOpsetOnly @@ -14,7 +19,7 @@ parameters: - 0 jobs: -- job: MacOS_C_API_Packaging_CPU_arm64 +- job: MacOS_C_API_Packaging_CPU_${{ parameters.MacosArch }} workspace: clean: all variables: @@ -60,7 +65,14 @@ jobs: echo "##vso[task.prependpath]$JAVA_HOME/bin" displayName: 'Install JDK 17' - - template: mac-cpu-packaging-steps.yml - parameters: - MacosArch: arm64 - AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} --build_java --build_nodejs --use_coreml --use_webgpu --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64 + - ${{ if eq(parameters.MacosArch, 'arm64') }}: + - template: mac-cpu-packaging-steps.yml + parameters: + MacosArch: ${{ parameters.MacosArch }} + AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} --build_java --build_nodejs --use_coreml --use_webgpu --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64 + + - ${{ if eq(parameters.MacosArch, 'x86_64') }}: + - template: mac-cpu-packaging-steps.yml + parameters: + MacosArch: ${{ parameters.MacosArch }} + AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} --build_java --build_nodejs --use_coreml --use_webgpu --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=x86_64 diff --git a/tools/ci_build/github/windows/jar_packaging.py b/tools/ci_build/github/windows/jar_packaging.py index f4bc6899260c1..22cc88e1c0666 100644 --- a/tools/ci_build/github/windows/jar_packaging.py +++ b/tools/ci_build/github/windows/jar_packaging.py @@ -232,6 +232,8 @@ def run_packaging(package_type: str, build_dir: str): "platforms": [ {"path": "onnxruntime-java-linux-x64", "lib": "libcustom_op_library.so", "archive_lib": True}, {"path": "onnxruntime-java-linux-aarch64", "lib": "libcustom_op_library.so", "archive_lib": False}, + # Archive only one osx dylib into testing.jar; add_file_to_archive writes by file name so a second entry would overwrite it. + {"path": "onnxruntime-java-osx-x86_64", "lib": "libcustom_op_library.dylib", "archive_lib": False}, {"path": "onnxruntime-java-osx-arm64", "lib": "libcustom_op_library.dylib", "archive_lib": True}, ] }, diff --git a/tools/ci_build/github/windows/jar_packaging_test.py b/tools/ci_build/github/windows/jar_packaging_test.py index e4f7e4945442c..470222ad54f08 100644 --- a/tools/ci_build/github/windows/jar_packaging_test.py +++ b/tools/ci_build/github/windows/jar_packaging_test.py @@ -54,16 +54,32 @@ def _setup_test_directory(package_type: str, version_string: str): # --- macOS and other platforms (for CPU test) --- if package_type == "cpu": - # Add linux-aarch64 and osx-arm64 for CPU test + # Add linux-aarch64, osx-x86_64, and osx-arm64 for CPU test linux_aarch64_dir = java_artifact_dir / "onnxruntime-java-linux-aarch64" linux_aarch64_native_dir = linux_aarch64_dir / "ai" / "onnxruntime" / "native" / "linux-aarch64" linux_aarch64_native_dir.mkdir(parents=True, exist_ok=True) create_empty_file(linux_aarch64_dir / "libcustom_op_library.so") + create_empty_file(linux_aarch64_native_dir / "libonnxruntime.so") + create_empty_file(linux_aarch64_native_dir / "libonnxruntime4j_jni.so") + + # The outer artifact directory keeps the CI arch tag (osx-x86_64 / + # osx-arm64) to match jar_packaging.py's platform list, but the inner + # ai/onnxruntime/native/ path uses the Java-convention arch + # names (osx-x64 / osx-aarch64) applied by + # linux_java_copy_strip_binary.py for .dylib builds. + osx_x86_64_dir = java_artifact_dir / "onnxruntime-java-osx-x86_64" + osx_x86_64_native_dir = osx_x86_64_dir / "ai" / "onnxruntime" / "native" / "osx-x64" + osx_x86_64_native_dir.mkdir(parents=True, exist_ok=True) + (osx_x86_64_dir / "libcustom_op_library.dylib").write_bytes(b"x86_64-custom-op-marker") + create_empty_file(osx_x86_64_native_dir / "libonnxruntime.dylib") + create_empty_file(osx_x86_64_native_dir / "libonnxruntime4j_jni.dylib") osx_arm64_dir = java_artifact_dir / "onnxruntime-java-osx-arm64" - osx_arm64_native_dir = osx_arm64_dir / "ai" / "onnxruntime" / "native" / "osx-arm64" + osx_arm64_native_dir = osx_arm64_dir / "ai" / "onnxruntime" / "native" / "osx-aarch64" osx_arm64_native_dir.mkdir(parents=True, exist_ok=True) - create_empty_file(osx_arm64_dir / "libcustom_op_library.dylib") + (osx_arm64_dir / "libcustom_op_library.dylib").write_bytes(b"arm64-custom-op-marker") + create_empty_file(osx_arm64_native_dir / "libonnxruntime.dylib") + create_empty_file(osx_arm64_native_dir / "libonnxruntime4j_jni.dylib") return tmp_path @@ -126,6 +142,17 @@ def test_cpu_packaging(directory_setup_factory, version_string): # Linux libs assert "ai/onnxruntime/native/linux-x64/libonnxruntime.so" in jar_contents assert "ai/onnxruntime/native/linux-x64/libonnxruntime4j_jni.so" in jar_contents + assert "ai/onnxruntime/native/linux-aarch64/libonnxruntime.so" in jar_contents + assert "ai/onnxruntime/native/linux-aarch64/libonnxruntime4j_jni.so" in jar_contents + # macOS libs -- under the Java-convention arch paths (osx-x64 / + # osx-aarch64) produced by linux_java_copy_strip_binary.py. + assert "ai/onnxruntime/native/osx-x64/libonnxruntime.dylib" in jar_contents + assert "ai/onnxruntime/native/osx-x64/libonnxruntime4j_jni.dylib" in jar_contents + assert "ai/onnxruntime/native/osx-aarch64/libonnxruntime.dylib" in jar_contents + assert "ai/onnxruntime/native/osx-aarch64/libonnxruntime4j_jni.dylib" in jar_contents + # The pre-rename CI-arch paths must NOT appear in the JAR. + assert "ai/onnxruntime/native/osx-x86_64/libonnxruntime.dylib" not in jar_contents + assert "ai/onnxruntime/native/osx-arm64/libonnxruntime.dylib" not in jar_contents # GPU libs should NOT be present assert "ai/onnxruntime/native/linux-x64/libonnxruntime_providers_cuda.so" not in jar_contents @@ -134,11 +161,17 @@ def test_cpu_packaging(directory_setup_factory, version_string): jar_contents = zf.namelist() assert "libcustom_op_library.so" in jar_contents assert "libcustom_op_library.dylib" in jar_contents + # Both osx platforms produce a same-named dylib; only arm64 must be archived so that + # its entry is not silently overwritten by a subsequent x86_64 archive call. + assert jar_contents.count("libcustom_op_library.dylib") == 1 + assert zf.read("libcustom_op_library.dylib") == b"arm64-custom-op-marker" # 3. Verify the custom op libraries were removed from the source directories linux_dir = temp_build_dir / "java-artifact" / "onnxruntime-java-linux-x64" linux_aarch64_dir = temp_build_dir / "java-artifact" / "onnxruntime-java-linux-aarch64" + osx_x86_64_dir = temp_build_dir / "java-artifact" / "onnxruntime-java-osx-x86_64" osx_arm64_dir = temp_build_dir / "java-artifact" / "onnxruntime-java-osx-arm64" assert not (linux_dir / "libcustom_op_library.so").exists() assert not (linux_aarch64_dir / "libcustom_op_library.so").exists() + assert not (osx_x86_64_dir / "libcustom_op_library.dylib").exists() assert not (osx_arm64_dir / "libcustom_op_library.dylib").exists()