ORT 1.24.3 release cherry pick round 2

.github/actions/setup-android-ndk/action.yml

@@ -89,10 +89,10 @@ runs:
           set -e -x
           python3 tools/python/run_android_emulator.py \
             --android-sdk-root "${ANDROID_SDK_ROOT}" \
-            --start --emulator-extra-args="-partition-size 2047" \
+            --start --emulator-extra-args="-partition-size 2047 -memory 5120" \
             --emulator-pid-file ./emulator.pid
           echo "Emulator PID: `cat ./emulator.pid`"
 
     - name: View Android ENVs
       shell: bash
-      run: env | grep ANDROID
+      run: env | grep ANDROID

onnxruntime/test/providers/cpu/tensor/gather_op_test.cc

@@ -342,37 +342,49 @@ TEST(GatherOpTest, Gather_axis1_indices2d_string) {
 }
 
 TEST(GatherOpTest, Gather_overflow_check) {
-// Skip on 32-bit platforms where size_t overflow would truncate the large expected
-// output shape and where allocating the full reference tensor is infeasible.
+  // Skip on 32-bit platforms where allocating the full reference tensor is infeasible due
+  // to std::vector::max_size being limited to the size of ptrdiff_t (INT32_MAX on 32-bit).
+  // Also, peak memory usage for this test would be greater than what is addressable.
 #if SIZE_MAX <= UINT32_MAX
   GTEST_SKIP() << "Gather_overflow_check skipped on 32-bit platforms.";
 #endif
 
-  // The test uses dimensions (65537, 2) and indices of length 65537, which produce an output
-  // shape of (65537, 65537).
+  // The test uses dimensions (46341, 2) and indices of length 46341, which produce an output
+  // shape of (46341, 46341).
   //
-  // 65537 x 65537 = 4,295,098,369 which is greater than the maximum value of a 32-bit integer (2,147,483,647).
+  // 46341 x 46341 = 2,147,488,281 which is just greater than the maximum value of a 32-bit integer (2,147,483,647).
   //
   // This test is to verify CPU implementation of the Gather operator doesn't overflow when calculating
   // the output shape and generating the output tensor.
 
+  constexpr int64_t dim_val = 46341;
+
   OpTester test("Gather");
   test.AddAttribute<int64_t>("axis", 1LL);
 
-  // Inputs
-  const std::vector<int64_t> data_dims{65537, 2};
-  const std::vector<int64_t> indices_dims{65537};
-  std::vector<uint8_t> data_values(static_cast<size_t>(data_dims[0] * data_dims[1]), 1);
-  std::vector<int64_t> indices_values(static_cast<size_t>(indices_dims[0]), 1);
-  std::vector<uint8_t> expected_output_values(static_cast<size_t>(65537) * static_cast<size_t>(65537), 1);
+  // Setup test inputs and outputs in a separate scope to ensure the large `expected_output_values` array
+  // is destroyed before we run the test via `test.Run()`.
+  {
+    const std::vector<int64_t> data_dims{dim_val, 2};
+    const std::vector<int64_t> indices_dims{dim_val};
+    std::vector<uint8_t> data_values(static_cast<size_t>(data_dims[0] * data_dims[1]), 1);
+    std::vector<int64_t> indices_values(static_cast<size_t>(indices_dims[0]), 1);
+    std::vector<uint8_t> expected_output_values(static_cast<size_t>(dim_val) * static_cast<size_t>(dim_val), 1);
+
+    test.AddInput<uint8_t>("data", {dim_val, 2}, data_values);
+    test.AddInput<int64_t>("indices", {dim_val}, indices_values);
 
-  test.AddInput<uint8_t>("data", {65537, 2}, data_values);
-  test.AddInput<int64_t>("indices", {65537}, indices_values);
-  test.AddOutput<uint8_t>("output", {65537, 65537}, expected_output_values);
+    // Note: the large ~2GiB `expected_output_values` array is copied into the OpTester.
+    test.AddOutput<uint8_t>("output", {dim_val, dim_val}, expected_output_values);
+  }
 
   std::vector<std::unique_ptr<IExecutionProvider>> execution_providers;
   execution_providers.emplace_back(DefaultCpuExecutionProvider());
 
+  // Note: peak memory usage will be in the order of multiple GiB:
+  //  - OpTester holds expected outputs buffer of size ~2GiB
+  //  - The session state allocates a buffer for the output of size ~2GiB
+  //  - Other overhead and bookkeeping.
   test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers);
 }
 

tools/ci_build/github/azure-pipelines/jar_package_testing.yml

@@ -4,173 +4,62 @@ resources:
     source: 'Zip-Nuget-Java-Nodejs Packaging Pipeline'
     trigger: true
     branch: main
+  repositories:
+  - repository: 1esPipelines
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
 
 variables:
   mavenVersion: '3.9.8'
 
-stages:
-- template: templates/final-jar-testing-win.yml
+extends:
+  # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
+  # For non-production pipelines, use "Unofficial" as defined below.
+  # For productions pipelines, use "Official".
+  template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
   parameters:
-    PoolName: 'onnxruntime-Win-CPU-VS2022-Latest'
-
-- template: templates/final-jar-testing-linux.yml
-  parameters:
-    OS: Linux
-    PoolName: 'onnxruntime-Ubuntu2204-AMD-CPU'
-
-- template: templates/final-jar-testing-linux.yml
-  parameters:
-    OS: MacOS
-    PoolName: 'AcesShared'
-    PoolDemands: 'ImageOverride -equals ACES_VM_SharedPool_Sequoia'
-
-- stage: GPU_JAR_Testing
-  dependsOn: []
-  jobs:
-  - job: Final_Jar_Testing_Windows_GPU
-    workspace:
-      clean: all
-    pool: 'onnxruntime-Win2022-GPU-A10'
-    timeoutInMinutes: 60
-    variables:
-    - name: runCodesignValidationInjection
-      value: false
-
-    steps:
-    - template: templates/set-version-number-variables-step.yml
-
-    - template: templates/jobs/download_win_gpu_library.yml
+    featureFlags:
+      binskimScanAllExtensions: true
+    sdl:
+      binskim:
+        enabled: true
+        scanOutputDirectoryOnly: true
+      sourceAnalysisPool:
+        name: onnxruntime-Win-CPU-VS2022-Latest
+        os: windows
+      componentgovernance:
+        ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)'
+      spotBugs:
+        enabled: false
+        justificationForDisabling: "Getting ##[error]1. SpotBugs Error gdn.unknownFormatResult - File: spotbugs.xml, which indicates that SpotBugs found one or more errors, which are not handled by the Guardian right now."
+      codeql:
+        compiled:
+          enabled: false
+          justificationForDisabling: 'CodeQL is taking nearly 6 hours resulting in timeouts in our production pipelines'
+      tsa:
+        enabled: true
+      codeSignValidation:
+        enabled: true
+        break: true
+      policheck:
+        enabled: true
+        exclusionsFile: '$(Build.SourcesDirectory)\tools\ci_build\policheck_exclusions.xml'
+
+    stages:
+    - template: templates/final-jar-testing-win.yml
       parameters:
-        CudaVersion: 12.8
-        DownloadCUDA: true
-        DownloadTRT: true
-
-    - template: templates/setup-maven.yml
-
-    - task: Maven@4
-      displayName: 'Download Java Dependencies'
-      inputs:
-        mavenPomFile: '$(Build.SourcesDirectory)/tools/ci_build/java/pom.xml'
-        goals: 'dependency:copy-dependencies'
-        options: '-DoutputDirectory=$(Pipeline.Workspace)/build/onnxruntime-java'
-        publishJUnitTestResults: false
-        javaHomeOption: 'JDKVersion'
-        jdkVersionOption: '1.17'
-        mavenVersionOption: 'Default'
-    - download: build
-      artifact: 'onnxruntime-java-gpu'
-      displayName: 'Download Final Jar'
-    - script: |
-        move $(Pipeline.Workspace)\build\onnxruntime-java-gpu\*.jar $(Pipeline.Workspace)\build\onnxruntime-java\
-
-    - task: PowerShell@2
-      displayName: 'Run Java Tests with PowerShell'
-      inputs:
-        targetType: 'inline'
-        script: |
-          # Exit script on any error
-          $ErrorActionPreference = "Stop"
-
-          cd $(Pipeline.Workspace)/build/onnxruntime-java
-          del *.asc
-          del *.sha256
-          del *.sha512
-          del *.pom
-          del *.sha1
-          del *.pom
-          cd ..
-          mkdir tests
-          cd tests
-          jar xf $(Pipeline.Workspace)/build/onnxruntime-java/testing.jar
-          del $(Pipeline.Workspace)/build/onnxruntime-java/testing.jar
-          dir $(Pipeline.Workspace)/build/tests
-          Write-Host "Running JUnit Tests..."
-          & java -DUSE_CUDA=1 `
-            -cp "$(Pipeline.Workspace)\build\tests;$(Pipeline.Workspace)\build\onnxruntime-java\*" org.junit.platform.console.ConsoleLauncher --scan-classpath=$(Pipeline.Workspace)\build\tests `
-            --fail-if-no-tests --disable-banner --reports-dir "$($env:Build_ArtifactStagingDirectory)/TestResults"
-
-    - task: PublishTestResults@2
-      displayName: 'Publish Test Results'
-      inputs:
-        testResultsFormat: 'JUnit'
-        testResultsFiles: '$(Build.ArtifactStagingDirectory)/TestResults/TEST-junit-jupiter.xml'
-        failTaskOnFailedTests: true
+        PoolName: 'onnxruntime-Win-CPU-VS2022-Latest'
 
+    - template: templates/final-jar-testing-linux.yml
+      parameters:
+        OS: linux
+        PoolName: 'onnxruntime-Ubuntu2204-AMD-CPU'
 
-  - job: Final_Jar_Testing_Linux_GPU
-    workspace:
-      clean: all
-    pool:
-      name: 'Onnxruntime-Linux-GPU-A10'
-    variables:
-    - name: runCodesignValidationInjection
-      value: false
-    - name: docker_base_image
-      value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_almalinux8_gcc14:20251017.1
-    timeoutInMinutes: 60
-    steps:
-    - checkout: self
-      submodules: false
-
-    - template: templates/set-version-number-variables-step.yml
-
-    - bash: |
-       sudo apt-get install -y msopenjdk-17
-       dpkg -l msopenjdk-17
-
-    - bash: |
-        echo "Downloading and installing Maven $(mavenVersion) for Linux..."
-        MAVEN_DIR="$(Agent.TempDirectory)/apache-maven-$(mavenVersion)"
-        # Download Maven binary
-        wget https://archive.apache.org/dist/maven/maven-3/$(mavenVersion)/binaries/apache-maven-$(mavenVersion)-bin.tar.gz -O $(Agent.TempDirectory)/maven.tar.gz
-
-        # Extract to the temp directory
-        mkdir -p ${MAVEN_DIR}
-        tar -xzf $(Agent.TempDirectory)/maven.tar.gz -C $(Agent.TempDirectory)
-
-        # Add Maven's bin directory to the PATH for subsequent tasks in the job
-        echo "##vso[task.prependpath]${MAVEN_DIR}/bin"
-      displayName: 'Install Maven (Linux)'
-
-    - script: |
-        echo "Maven is now on the PATH."
-        mvn --version
-
-    - download: build
-      artifact: 'onnxruntime-java-gpu'
-      displayName: 'Download Final Jar'
-
-    # Rename the downloaded folder
-    - script: |
-        mv $(Pipeline.Workspace)/build/onnxruntime-java-gpu $(Pipeline.Workspace)/build/onnxruntime-java
-
-    - task: Maven@4
-      displayName: 'Download Dependencies'
-      inputs:
-        mavenPomFile: '$(Build.SourcesDirectory)/tools/ci_build/java/pom.xml'
-        goals: 'dependency:copy-dependencies'
-        options: '-DoutputDirectory=$(Pipeline.Workspace)/build/onnxruntime-java'
-        publishJUnitTestResults: false
-        javaHomeOption: 'Path'
-        jdkDirectory: '/usr/lib/jvm/msopenjdk-17-amd64'
-        jdkVersionOption: 'Default'
-        mavenVersionOption: 'Default'
-
-    # Now all the jars are in the $(Pipeline.Workspace)/build folder
-
-    - template: templates/get-docker-image-steps.yml
+    - template: templates/final-jar-testing-linux.yml
       parameters:
-        Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0
-        Context: tools/ci_build/github/linux/docker/
-        DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{ variables.docker_base_image }} --build-arg TRT_VERSION=${{ variables.linux_trt_version }}"
-        Repository: onnxruntimeubi8packagestest
+        OS: macOS
+        PoolName: 'AcesShared'
+        PoolDemands: 'ImageOverride -equals ACES_VM_SharedPool_Sequoia'
 
-    - bash: |
-        docker run --network=none --rm \
-          --gpus all \
-          --volume $(Build.SourcesDirectory):/onnxruntime_src \
-          --volume $(Pipeline.Workspace)/build:/build \
-          --volume /data/models:/build/models:ro \
-          onnxruntimeubi8packagestest \
-          /bin/bash /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -r /build -v $(OnnxRuntimeVersion)
-      displayName: 'Test'
+    - template: templates/final-jar-testing-gpu.yml