Migrate device and WASM tests from XHarness to DeviceRunners dotnet test

.config/dotnet-tools.json

@@ -23,13 +23,6 @@
       ],
       "rollForward": false
     },
-    "microsoft.dotnet.xharness.cli": {
-      "version": "11.0.0-prerelease.26107.1",
-      "commands": [
-        "xharness"
-      ],
-      "rollForward": false
-    },
     "api-tools": {
       "version": "1.4.0",
       "commands": [
@@ -50,6 +43,13 @@
         "docfx"
       ],
       "rollForward": false
+    },
+    "appledev.tools": {
+      "version": "0.8.10",
+      "commands": [
+        "apple"
+      ],
+      "rollForward": false
     }
   }
-}
+}

.gitignore

@@ -323,3 +323,4 @@ fastlane/screenshots
 .docs
 .playwright-mcp/
 .github/aw/logs/
+tests/SkiaSharp.Tests.Wasm/test-results/

documentation/dev/updating-dotnet-version.md

@@ -75,7 +75,6 @@ All use `$(TFMPrevious)-platform$(TPVPrevious);$(TFMCurrent)-platform$(TPVCurren
 ### 7. Utility Projects
 
 - [ ] `utils/SkiaSharpGenerator/SkiaSharpGenerator.csproj`
-- [ ] `utils/WasmTestRunner/WasmTestRunner.csproj`
 - [ ] `utils/NativeLibraryMiniTest/docker/NativeLibraryMiniTest.csproj`
 
 ### 8. Sample Projects

nuget.config

@@ -2,6 +2,7 @@
 <configuration>
   <packageSources>
     <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
     <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
   </packageSources>

scripts/azure-templates-stages-test.yml

@@ -246,8 +246,8 @@ stages:
                 displayName: Publish the Android test results
                 condition: always()
                 inputs:
-                  testResultsFormat: xUnit
-                  testResultsFiles: 'output/logs/testlogs/**/TestResults.xml'
+                  testResultsFormat: VSTest
+                  testResultsFiles: 'output/logs/testlogs/**/*.trx'
                   testRunTitle: 'Android Tests (API $(ANDROID_TEST_DEVICE_VERSION))'
             publishArtifacts:
               - name: testlogs_android_$(ANDROID_TEST_DEVICE_VERSION)
@@ -270,8 +270,8 @@ stages:
                 displayName: Publish the iOS test results
                 condition: always()
                 inputs:
-                  testResultsFormat: xUnit
-                  testResultsFiles: 'output/logs/testlogs/**/TestResults.xml'
+                  testResultsFormat: VSTest
+                  testResultsFiles: 'output/logs/testlogs/**/*.trx'
                   testRunTitle: 'iOS Tests (v$(IOS_TEST_DEVICE_VERSION))'
             publishArtifacts:
               - name: testlogs_ios_$(IOS_TEST_DEVICE_VERSION)
@@ -296,8 +296,8 @@ stages:
                 displayName: Publish the Mac Catalyst test results
                 condition: always()
                 inputs:
-                  testResultsFormat: xUnit
-                  testResultsFiles: 'output/logs/testlogs/**/TestResults.xml'
+                  testResultsFormat: VSTest
+                  testResultsFiles: 'output/logs/testlogs/**/*.trx'
                   testRunTitle: 'Mac Catalyst Tests'
             publishArtifacts:
               - name: testlogs_maccatalyst
@@ -310,7 +310,7 @@ stages:
             buildAgent: ${{ parameters.buildAgentLinux }}
             packages: $(MANAGED_LINUX_PACKAGES) ninja-build
             target: tests-wasm
-            additionalArgs: --skipExternals="all" --coverage=false --chromedriver=$(CHROMEWEBDRIVER)
+            additionalArgs: --skipExternals="all" --coverage=false
             installAndroidSdk: false
             shouldPublish: false
             requiredArtifacts:
@@ -322,8 +322,8 @@ stages:
                 displayName: Publish the WASM test results
                 condition: always()
                 inputs:
-                  testResultsFormat: xUnit
-                  testResultsFiles: 'output/logs/testlogs/**/*.xml'
+                  testResultsFormat: VSTest
+                  testResultsFiles: 'output/logs/testlogs/**/*.trx'
                   testRunTitle: 'Linux WASM Tests'
             publishArtifacts:
               - name: testlogs_wasm

scripts/infra/caching/repo-deps.json

@@ -111,7 +111,6 @@
             "tests/**",
             "scripts/infra/tests/**",
             "benchmarks/**",
-            "utils/WasmTestRunner/**",
             "utils/NativeLibraryMiniTest/**"
           ]
         }

scripts/infra/shared/shared.cake

@@ -300,5 +300,9 @@ void TakeSnapshot(DirectoryPath output, string name)
     var fname = $"screenshot-{DateTime.Now:yyyyMMdd_hhmmss}-{name}.jpg";
     var dest = output.CombineWithFilePath(fname);
 
-    RunProcess("screencapture", dest.FullPath);
+    try {
+        RunProcess("screencapture", dest.FullPath);
+    } catch (Exception ex) {
+        Warning("Failed to take screenshot: {0}", ex.Message);
+    }
 }

scripts/infra/tests/test-shared.cake

@@ -2,6 +2,46 @@
 
 #tool nuget:?package=xunit.runner.console&version=2.4.2
 
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// DEVICE RUNNERS — shared helper for DeviceRunners.Testing.Targets based tests
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void RunDeviceRunnersTest(
+    FilePath testProject,
+    DirectoryPath output,
+    string configuration = null,
+    string framework = null,
+    bool noBuild = false,
+    Dictionary<string, string> properties = null)
+{
+    CleanDirectories($"{PACKAGE_CACHE_PATH}/skiasharp*");
+    CleanDirectories($"{PACKAGE_CACHE_PATH}/harfbuzzsharp*");
+
+    output = MakeAbsolute(output);
+    CleanDirectories(output.FullPath);
+
+    var settings = new DotNetTestSettings {
+        Configuration = configuration ?? CONFIGURATION,
+        Framework = framework,
+        NoBuild = noBuild,
+        ResultsDirectory = output,
+        Verbosity = DotNetVerbosity.Normal,
+        ArgumentCustomization = args => {
+            args = args.Append("--logger").Append("trx");
+            if (properties != null) {
+                foreach (var prop in properties) {
+                    if (!string.IsNullOrEmpty(prop.Value)) {
+                        args = args.Append($"/p:{prop.Key}={prop.Value}");
+                    }
+                }
+            }
+            return args;
+        },
+    };
+
+    DotNetTest(MakeAbsolute(testProject).FullPath, settings);
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 // TEST UTILITIES — shared by desktop test cakes
 ////////////////////////////////////////////////////////////////////////////////////////////////////

scripts/infra/tests/tests-android.cake

@@ -2,17 +2,15 @@ DirectoryPath ROOT_PATH = MakeAbsolute(Directory("../../.."));
 
 #load "../shared/shared.cake"
 #load "../shared/msbuild.cake"
+#load "test-shared.cake"
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
-// ANDROID TESTS — build, emulator management, and xharness execution
+// ANDROID TESTS — build, emulator management, and dotnet test execution
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-var TEST_APP = Argument("app", EnvironmentVariable("ANDROID_TEST_APP") ?? "");
 var TEST_RESULTS = Argument("results", EnvironmentVariable("ANDROID_TEST_RESULTS") ?? "");
 var TEST_DEVICE = Argument("device", EnvironmentVariable("ANDROID_TEST_DEVICE") ?? "android-emulator-64");
 var TEST_VERSION = Argument("deviceVersion", EnvironmentVariable("ANDROID_TEST_DEVICE_VERSION") ?? "36");
-var TEST_APP_PACKAGE_NAME = Argument("package", EnvironmentVariable("ANDROID_TEST_APP_PACKAGE_NAME") ?? "");
-var TEST_APP_INSTRUMENTATION = Argument("instrumentation", EnvironmentVariable("ANDROID_TEST_APP_INSTRUMENTATION") ?? "devicerunners.xharness.maui.XHarnessInstrumentation");
 
 var ANDROID_AVD = "DEVICE_TESTS_EMULATOR";
 var DEVICE_NAME = Argument("skin", EnvironmentVariable("ANDROID_TEST_SKIN") ?? "Nexus 5X");
@@ -22,37 +20,6 @@ var usingEmulator = true;
 
 Setup(context =>
 {
-    // if app wasn't passed as argument, build it
-    if (string.IsNullOrEmpty(TEST_APP)) {
-        FilePath csproj = $"{ROOT_PATH}/tests/SkiaSharp.Tests.Devices/SkiaSharp.Tests.Devices.csproj";
-        var configuration = "Release";
-        var tfm = "net10.0-android";
-        var rid = "android-" + RuntimeInformation.ProcessArchitecture.ToString().ToLower();
-        TEST_APP = ROOT_PATH + $"/tests/SkiaSharp.Tests.Devices/bin/{configuration}/{tfm}/{rid}/com.companyname.SkiaSharpTests-Signed.apk";
-
-        Information("=== Android Test Build Configuration ===");
-        Information("  Project:       {0}", csproj);
-        Information("  Configuration: {0}", configuration);
-        Information("  TFM:           {0}", tfm);
-        Information("  RID:           {0}", rid);
-        Information("  App Path:      {0}", TEST_APP);
-        Information("  OS:            {0}", RuntimeInformation.OSDescription);
-        Information("  Arch:          {0}", RuntimeInformation.ProcessArchitecture);
-        Information("========================================");
-
-        CleanDirectories($"{PACKAGE_CACHE_PATH}/skiasharp*");
-        CleanDirectories($"{PACKAGE_CACHE_PATH}/harfbuzzsharp*");
-
-        if (!SKIP_BUILD) {
-            RunDotNetBuild(csproj,
-                configuration: configuration,
-                properties: new Dictionary<string, string> {
-                    { "TargetFramework", tfm },
-                    { "RuntimeIdentifier", rid },
-                });
-        }
-    }
-
     if (string.IsNullOrEmpty(TEST_RESULTS)) {
         TEST_RESULTS = $"{ROOT_PATH}/output/logs/testlogs/SkiaSharp.Tests.Devices.Android/{DATE_TIME_STR}";
     }
@@ -62,7 +29,6 @@ Setup(context =>
     if (!string.IsNullOrEmpty(TEST_VERSION) && TEST_VERSION != "latest")
         TEST_DEVICE = $"{TEST_DEVICE}_{TEST_VERSION}";
 
-    Information("Test App: {0}", TEST_APP);
     Information("Test Device: {0}", TEST_DEVICE);
 
     // determine the device characteristics
@@ -91,6 +57,17 @@ Setup(context =>
         DEVICE_ID = $"system-images;android-{api};google_apis;{DEVICE_ARCH}";
     }
 
+    // Pre-build the project before starting the emulator to reduce peak memory usage.
+    // Android IL linking is very memory-intensive and running it concurrently with the
+    // emulator can cause OOM on CI agents.
+    Information("Pre-building Android test project...");
+    FilePath csproj = $"{ROOT_PATH}/tests/SkiaSharp.Tests.Devices/SkiaSharp.Tests.Devices.csproj";
+    DotNetBuild(MakeAbsolute(csproj).FullPath, new DotNetBuildSettings {
+        Configuration = "Release",
+        Framework = "net10.0-android",
+    });
+    Information("Pre-build complete.");
+
     if (!usingEmulator) {
         Information("Using a physical device:");
         DotNetTool("android device list");
@@ -117,6 +94,15 @@ Setup(context =>
     Information("Emulator started:");
     DotNetTool("android device list");
 
+    // Set up adb reverse so the app can connect to the host's TCP listener
+    // via localhost (the CLI's env injects DEVICE_RUNNERS_HOST_NAMES=localhost;10.0.2.2)
+    Information("Setting up adb reverse for TCP port forwarding...");
+    var androidHome = EnvironmentVariable("ANDROID_HOME") ?? EnvironmentVariable("ANDROID_SDK_ROOT");
+    var adb = androidHome != null
+        ? $"{androidHome}/platform-tools/adb"
+        : "adb";
+    StartProcess(adb, "reverse tcp:16384 tcp:16384");
+
     TakeSnapshot(TEST_RESULTS, "boot-complete");
 });
 
@@ -132,47 +118,9 @@ Teardown(context =>
 Task("Default")
     .Does(() =>
 {
-    if (string.IsNullOrEmpty(TEST_APP_PACKAGE_NAME)) {
-        var appFile = (FilePath)TEST_APP;
-        appFile = appFile.GetFilenameWithoutExtension();
-        TEST_APP_PACKAGE_NAME = appFile.FullPath.Replace("-Signed", "");
-    }
-    if (string.IsNullOrEmpty(TEST_APP_INSTRUMENTATION)) {
-        TEST_APP_INSTRUMENTATION = TEST_APP_PACKAGE_NAME + ".TestInstrumentation";
-    }
-
-    Information("Test App: {0}", TEST_APP);
-    Information("Test App Package Name: {0}", TEST_APP_PACKAGE_NAME);
-    Information("Test App Instrumentation: {0}", TEST_APP_INSTRUMENTATION);
-    Information("Test Results Directory: {0}", TEST_RESULTS);
-
-    TakeSnapshot(TEST_RESULTS, "starting-tests");
-
-    var complete = false;
-    System.Threading.Tasks.Task.Run(() => {
-        while (!complete) {
-            TakeSnapshot(TEST_RESULTS, "running-tests");
-            System.Threading.Thread.Sleep(5000);
-        }
-    });
+    FilePath csproj = $"{ROOT_PATH}/tests/SkiaSharp.Tests.Devices/SkiaSharp.Tests.Devices.csproj";
 
-    DotNetTool("xharness android test " +
-        $"--app=\"{TEST_APP}\" " +
-        $"--package-name=\"{TEST_APP_PACKAGE_NAME}\" " +
-        $"--instrumentation=\"{TEST_APP_INSTRUMENTATION}\" " +
-        $"--output-directory=\"{TEST_RESULTS}\" " +
-        $"--timeout=00:15:00 " +
-        $"--launch-timeout=00:05:00 " +
-        $"--verbosity=\"Debug\" ");
-
-    complete = true;
-
-    TakeSnapshot(TEST_RESULTS, "finished-tests");
-
-    var failed = XmlPeek($"{TEST_RESULTS}/TestResults.xml", "/assemblies/assembly[@failed > 0 or @errors > 0]/@failed");
-    if (!string.IsNullOrEmpty(failed)) {
-        throw new Exception($"At least {failed} test(s) failed.");
-    }
+    RunDeviceRunnersTest(csproj, (DirectoryPath)TEST_RESULTS, configuration: "Release", framework: "net10.0-android", noBuild: true);
 });
 
 RunTarget(TARGET);