[mobile] Add Android NuGet BrowserStack test to NuGet packaging pipeline

csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj

@@ -50,6 +50,7 @@
 
     <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.Common\InferenceTest.cs" />
     <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.Common\EqualityComparers.cs" />
+    <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.Common\AssertUtils.cs" />
     <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.Common\OnnxMl.cs" />
     <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.Common\OnnxData.cs" />
     <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\InferenceTest.netcore.cs" />

csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/BrowserStackTest.cs

@@ -45,8 +45,8 @@ public void Dispose()
                 {
                     String failureMessage = TestContext.CurrentContext.Result.Message;
                     String jsonToSendFailure =
-                        String.Format("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": " +
-                                      "{\"status\":\"failed\", \"reason\": {0}}}",
+                        String.Format("browserstack_executor: {{\"action\": \"setSessionStatus\", \"arguments\": " +
+                                      "{{\"status\":\"failed\", \"reason\": {0}}}}}",
                                       JsonConvert.ToString(failureMessage));
 
                     ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSendFailure);

csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/RunAllTest.cs

@@ -89,7 +89,7 @@ public async Task ClickRunAllTest()
                 await Task.Delay(500);
             }
 
-            var (numPassed, numFailed) = GetPassFailCount();
+            (int numPassed, int numFailed) = GetPassFailCount();
 
             if (numFailed == 0)
             {

csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/AssertUtils.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Xunit;
+
+namespace Microsoft.ML.OnnxRuntime.Tests
+{
+  internal static class AssertUtils
+  {
+
+    /// <summary>
+    /// Check if the action throws the expected exception. If it doesn't, the method passes. If it does, check for
+    /// the exception type and the expected exception message. More detailed Assert method to be used for unit tests
+    /// written with XUnit.
+    /// </summary>
+    /// <typeparam name="T">Type of exception expected to be thrown.</typeparam>
+    /// <param name="action">Action to be executed or tested.</param>
+    /// <param name="feedbackMessage">Feedback message if an unexpected exception happens.</param>
+    /// <param name="expectedExceptionMessage">Expected exception message. If null, the exception message is not
+    // checked.</param>
+    public static void IfThrowsCheckException<T>(Action action, string feedbackMessage, string expectedExceptionMessage = null) where T : Exception
+    {
+      try
+      {
+        action();
+      }
+      catch (T ex)
+      {
+        if (expectedExceptionMessage == null)
+        {
+          return;
+        }
+        else
+        {
+          Assert.True(ex.Message.Contains(expectedExceptionMessage),
+            $"{feedbackMessage}\nExpected exception message to contain '{expectedExceptionMessage}', but got '{ex.Message}'");
+        }
+      }
+      catch (Exception ex)
+      {
+        Assert.Fail($"{feedbackMessage}\nExpected {typeof(T).Name} but got {ex.GetType().Name}. ");
+      }
+    }
+
+
+    /// <summary>
+    /// Check if the action throws the expected exception. If it doesn't, the method fails with the feedbackMessage.
+    /// If it does, check for the exception type and the expected exception message. More detailed Assert method to be
+    /// used for unit tests written with XUnit.
+    /// </summary>
+    /// <typeparam name="T">Type of exception expected to be thrown.</typeparam>
+    /// <param name="action">Action to be executed or tested. It is expected that the action will throw.</param>
+    /// <param name="feedbackMessage">Feedback message if an unexpected exception happens.</param>
+    /// <param name="expectedExceptionMessage">Expected exception message. If null, the exception message is not
+    // checked.</param>
+    public static void AssertThrowsCheckException<T>(Action action, string feedbackMessage, string expectedExceptionMessage = null) where T : Exception
+    {
+      try
+      {
+        action();
+        Assert.Fail($"{feedbackMessage}\nExpected {typeof(T).Name} but no exception was thrown.");
+      }
+      catch (T ex)
+      {
+        if (expectedExceptionMessage == null)
+        {
+          return;
+        }
+        else
+        {
+          Assert.True(ex.Message.Contains(expectedExceptionMessage),
+            $"{feedbackMessage}\nExpected exception message to contain '{expectedExceptionMessage}', but got '{ex.Message}'");
+        }
+      }
+      catch (Exception ex)
+      {
+        Assert.Fail($"{feedbackMessage}\nExpected {typeof(T).Name} but got {ex.GetType().Name}. ");
+      }
+    }
+  }
+}

csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs

@@ -93,18 +93,35 @@ public void TestSessionOptions()
                 opt.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_EXTENDED;
                 Assert.Equal(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, opt.GraphOptimizationLevel);
 
-                Assert.Throws<OnnxRuntimeException>(() => { opt.GraphOptimizationLevel = (GraphOptimizationLevel)10; });
+                AssertUtils.AssertThrowsCheckException<OnnxRuntimeException>(
+                    () => { opt.GraphOptimizationLevel = (GraphOptimizationLevel)10; },
+                    "Set an invalid Graph Optimization Level.");
 
                 opt.AddSessionConfigEntry("key", "value");
 
-                var ex = Assert.Throws<OnnxRuntimeException>(() => { opt.AddSessionConfigEntry("", "invalid key"); });
-                Assert.Contains("[ErrorCode:InvalidArgument] Config key is empty", ex.Message);
+                AssertUtils.AssertThrowsCheckException<OnnxRuntimeException>(
+                    () => { opt.AddSessionConfigEntry("", "invalid key"); },
+                    "Added an invalid config entry.",
+                    "[ErrorCode:InvalidArgument] Config key is empty");
 
                 // SessionOptions.RegisterOrtExtensions can be manually tested by referencing the
                 // Microsoft.ML.OnnxRuntime.Extensions nuget package. After that is done, this should not throw.
-                ex = Assert.Throws<OnnxRuntimeException>(() => { opt.RegisterOrtExtensions(); });
-                Assert.Contains("Microsoft.ML.OnnxRuntime.Extensions NuGet package must be referenced", ex.Message);
-
+                AssertUtils.AssertThrowsCheckException<OnnxRuntimeException>(
+                    () => { opt.RegisterOrtExtensions(); },
+                    "RegisterOrtExtensions should throw if the Extensions package is not referenced",
+                    "Microsoft.ML.OnnxRuntime.Extensions NuGet package must be referenced");
+
+                // The below tests what happens when various execution providers are added
+                // to the session options.
+
+                // We can only check what EPs the package was built with for the
+                // Microsoft.ML.OnnxRuntime.Managed package because the managed package defines
+                // the C# preprocessor symbols (such as USE_CUDA) for the EPs that it was built with.
+
+                // The Microsoft.ML.OnnxRuntime package will use the appropriate platform bindings
+                // (ie the native Android bindings) where the C# preprocessor symbols
+                // identifying the EPs included in the build may not be available, so we use
+                // IfThrowsCheckException instead of using ifdefs.
 #if USE_CUDA
                 opt.AppendExecutionProvider_CUDA(0);
 #endif
@@ -157,30 +174,25 @@ public void TestSessionOptions()
 #if USE_TENSORRT
                 opt.AppendExecutionProvider_Tensorrt(0);
 #endif
-#if USE_XNNPACK
-                opt.AppendExecutionProvider("XNNPACK");
-#else
-                ex = Assert.Throws<OnnxRuntimeException>(() => { opt.AppendExecutionProvider("XNNPACK"); });
-                Assert.Contains("XNNPACK execution provider is not supported in this build", ex.Message);
-#endif
-#if USE_SNPE
-                opt.AppendExecutionProvider("SNPE");
-#else
-                ex = Assert.Throws<OnnxRuntimeException>(() => { opt.AppendExecutionProvider("SNPE"); });
-                Assert.Contains("SNPE execution provider is not supported in this build", ex.Message);
-#endif
-#if USE_QNN
-                opt.AppendExecutionProvider("QNN");
-#else
-                ex = Assert.Throws<OnnxRuntimeException>(() => { opt.AppendExecutionProvider("QNN"); });
-                Assert.Contains("QNN execution provider is not supported in this build", ex.Message);
-#endif
-#if USE_COREML
-                opt.AppendExecutionProvider("CoreML");
-#else
-                ex = Assert.Throws<OnnxRuntimeException>(() => { opt.AppendExecutionProvider("CoreML"); });
-                Assert.Contains("CoreML execution provider is not supported in this build", ex.Message);
-#endif
+                AssertUtils.IfThrowsCheckException<OnnxRuntimeException>(
+                    () => { opt.AppendExecutionProvider("CoreML"); },
+                    "Appending CoreML EP should have succeeded or thrown an OnnRuntimeException with the expected message. ",
+                    "CoreML execution provider is not supported in this build");
+
+                AssertUtils.IfThrowsCheckException<OnnxRuntimeException>(
+                    () => { opt.AppendExecutionProvider("XNNPACK"); },
+                    "Appending XNNPACK EP should have succeeded or thrown an OnnRuntimeException with the expected message. ",
+                    "XNNPACK execution provider is not supported in this build");
+
+                AssertUtils.IfThrowsCheckException<OnnxRuntimeException>(
+                    () => { opt.AppendExecutionProvider("SNPE"); },
+                    "Appending SNPE EP should have succeeded or thrown an OnnRuntimeException with the expected message. ",
+                    "SNPE execution provider is not supported in this build");
+
+                AssertUtils.IfThrowsCheckException<OnnxRuntimeException>(
+                    () => { opt.AppendExecutionProvider("QNN"); },
+                    "Appending QNN EP should have succeeded or thrown an OnnRuntimeException with the expected message. ",
+                    "QNN execution provider is not supported in this build");
 
                 opt.AppendExecutionProvider_CPU(1);
             }

tools/ci_build/github/azure-pipelines/nuget/templates/test_android.yml

@@ -0,0 +1,63 @@
+parameters:
+  AgentPool : 'Win-CPU'
+  ArtifactSuffix: ''
+  SpecificArtifact: false
+  BuildId: ''
+
+stages:
+- stage: NuGet_Test_Android
+  jobs:
+  - job:  NuGet_Test_Android
+    workspace:
+      clean: all
+    pool: "${{ parameters.AgentPool }}"
+
+    variables:
+    - name: OnnxRuntimeBuildDirectory
+      value: '$(Build.BinariesDirectory)'
+
+    steps:
+      - task: NuGetToolInstaller@0
+        displayName: Use Nuget 6.10.x
+        inputs:
+          versionSpec: 6.10.x
+
+      - template: ../../templates/flex-downloadPipelineArtifact.yml
+        parameters:
+          StepName: 'Download Pipeline Artifact'
+          ArtifactName: drop-signed-nuget-${{ parameters.ArtifactSuffix }}
+          TargetPath: '$(Build.BinariesDirectory)\nuget-artifact'
+          SpecificArtifact: ${{ parameters.SpecificArtifact }}
+          BuildId: ${{ parameters.BuildId }}
+
+      - template: get-nuget-package-version-as-variable.yml
+        parameters:
+          packageFolder: '$(Build.BinariesDirectory)\nuget-artifact'
+
+      - task: PowerShell@2
+        displayName: Install MAUI workloads
+        inputs:
+          targetType: 'inline'
+          script: |
+            dotnet workload install maui maui-android android
+          workingDirectory: '$(Build.SourcesDirectory)\csharp'
+
+      - task: PowerShell@2
+        displayName: Publish Android MAUI APK
+        inputs:
+          targetType: 'inline'
+          script: |
+            dotnet nuget add source $(Build.BinariesDirectory)\nuget-artifact --name local-nuget
+            dotnet publish -c Release --property:UsePrebuiltNativePackage=true --property:CurrentOnnxRuntimeVersion=$(NuGetPackageVersionNumber) -f net8.0-android
+          workingDirectory: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.MAUI'
+
+      - task: PowerShell@2
+        displayName: Run BrowserStack test
+        inputs:
+          targetType: 'inline'
+          script: |
+            dotnet test
+          workingDirectory: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android'
+        env:
+          BROWSERSTACK_USERNAME: $(browserstack_username)
+          BROWSERSTACK_ACCESS_KEY: $(browserstack_access_key)

tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml

@@ -24,8 +24,6 @@ jobs:
   pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
   workspace:
     clean: all
-  pool:
-    vmImage: 'macOS-14'
   variables:
     runCodesignValidationInjection: false
     ANDROID_AVD_HOME: $(Agent.TempDirectory)
@@ -114,4 +112,3 @@ jobs:
   - template: component-governance-component-detection-steps.yml
     parameters :
       condition : 'succeeded'
-

tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml

@@ -778,6 +778,13 @@ stages:
     SpecificArtifact: ${{ parameters.SpecificArtifact }}
     BuildId: ${{ parameters.BuildId }}
 
+- template: ../nuget/templates/test_android.yml
+  parameters:
+    AgentPool : 'onnxruntime-Win-CPU-2022'
+    ArtifactSuffix: 'CPU'
+    SpecificArtifact: ${{ parameters.SpecificArtifact }}
+    BuildId: ${{ parameters.BuildId }}
+
 - template: ../nuget/templates/test_linux.yml
   parameters:
     AgentPool : onnxruntime-Ubuntu2204-AMD-CPU