diff --git a/.github/instructions/helix-device-tests.instructions.md b/.github/instructions/helix-device-tests.instructions.md new file mode 100644 index 000000000000..354984c36f46 --- /dev/null +++ b/.github/instructions/helix-device-tests.instructions.md @@ -0,0 +1,162 @@ +--- +applyTo: "eng/helix_xharness.proj,eng/pipelines/**/device-tests*.yml,eng/pipelines/**/stage-device-tests.yml,src/**/DeviceTests/**" +description: "Guidelines for running and configuring .NET MAUI device tests on Helix infrastructure" +--- + +# Helix Device Tests Guidelines + +This document provides guidance for working with .NET MAUI device tests that run on Helix infrastructure using XHarness. + +## Overview + +.NET MAUI uses [.NET Engineering Services Helix](https://helix.dot.net) with XHarness to run device tests across multiple platforms in parallel. This provides cloud-based device testing infrastructure. + +### Device Test Projects + +- `Controls.DeviceTests` - UI control tests +- `Core.DeviceTests` - Core framework tests +- `Graphics.DeviceTests` - Graphics and drawing tests +- `Essentials.DeviceTests` - Platform API tests +- `MauiBlazorWebView.DeviceTests` - Blazor WebView tests + +### Available Helix Queues + +Current configuration uses: +- **iOS**: `osx.15.arm64.maui.open` +- **Mac Catalyst**: `osx.15.arm64.maui.open` +- **Android**: `ubuntu.2204.amd64.android.33.open` + +Check available queues at [helix.dot.net](https://helix.dot.net). + +## Key Configuration Files + +| File | Purpose | +|------|---------| +| `eng/helix_xharness.proj` | Main Helix configuration - defines scenarios, queues, and work items | +| `eng/pipelines/common/stage-device-tests.yml` | Pipeline template for device tests | +| `eng/test-configuration.json` | Test retry configuration | + +## iOS Category Splitting + +For iOS, Controls.DeviceTests heavy categories are split into separate Helix work items. This mirrors the old cake-based approach and enables parallel execution for the slowest tests. + +**How it works:** +1. Heavy categories are defined in `ControlsTestCategoriesToSkipForRestOfTests` property in `helix_xharness.proj` +2. The `ControlsTestCategoriesToRunIndividually` ItemGroup is populated from that property +3. Each heavy category becomes a separate Helix work item +4. All other Controls tests run together in a single "General" work item +5. XHarness passes `--set-env="TestFilter=Category=X"` for individual categories +6. XHarness passes `--set-env="TestFilter=SkipCategories=X;Y;Z"` for the "General" work item +7. Core.DeviceTests runs as a single work item (no splitting) + +**Heavy categories that run separately:** +- CollectionView, Shell, HybridWebView + +**Keep in sync:** If adding new heavy categories, update the `ControlsTestCategoriesToSkipForRestOfTests` property in `eng/helix_xharness.proj`. + +## Running Device Tests Locally + +### Prerequisites + +From the repository root: + +```bash +# 1. Restore dotnet tools +dotnet tool restore + +# 2. Build MSBuild tasks (required) +./build.sh -restore -build -configuration Release -projects $(PWD)/Microsoft.Maui.BuildTasks.slnf /bl:BuildBuildTasks.binlog -warnAsError false + +# 3. Build device tests +./build.sh -restore -build -configuration Release /p:BuildDeviceTests=true /bl:BuildDeviceTests.binlog -warnAsError false +``` + +### Submit to Helix + +Set required environment variables: + +```bash +export BUILD_REASON=pr +export BUILD_REPOSITORY_NAME=maui +export BUILD_SOURCEBRANCH=main +export SYSTEM_TEAMPROJECT=dnceng +export SYSTEM_ACCESSTOKEN='' +``` + +Submit tests: + +```bash +# Android +./eng/common/msbuild.sh ./eng/helix_xharness.proj /restore /p:TreatWarningsAsErrors=false /t:Test /p:TargetOS=android /bl:sendhelix_android.binlog + +# iOS +./eng/common/msbuild.sh ./eng/helix_xharness.proj /restore /p:TreatWarningsAsErrors=false /t:Test /p:TargetOS=ios /bl:sendhelix_ios.binlog + +# Mac Catalyst +./eng/common/msbuild.sh ./eng/helix_xharness.proj /restore /p:TreatWarningsAsErrors=false /t:Test /p:TargetOS=maccatalyst /bl:sendhelix_catalyst.binlog +``` + +### Windows Commands + +```cmd +set BUILD_REASON=pr +set BUILD_REPOSITORY_NAME=maui +set BUILD_SOURCEBRANCH=main +set SYSTEM_TEAMPROJECT=dnceng +set SYSTEM_ACCESSTOKEN= + +.\build.cmd -restore -build -configuration Release -projects ".\Microsoft.Maui.BuildTasks.slnf" /bl:BuildBuildTasks.binlog -warnAsError false +.\build.cmd -restore -build -configuration Release /p:BuildDeviceTests=true /bl:BuildDeviceTests.binlog -warnAsError false +.\eng\common\msbuild.cmd .\eng\helix_xharness.proj /restore /p:TreatWarningsAsErrors=false /t:Test /p:TargetOS=android /bl:sendhelix.binlog +``` + +### Validate MSBuild Logic Only + +To validate the helix proj without submitting (requires built artifacts): + +```bash +dotnet msbuild eng/helix_xharness.proj /t:DiscoverTestBundles /p:TargetOS=ios /p:_MauiDotNetTfm=net10.0 /p:RepoRoot=$(pwd)/ -v:n +``` + +## Configuration Details + +The `eng/helix_xharness.proj` configuration includes: + +- **Timeouts**: 2-hour work item timeout, 1-hour 15-min test timeout for category splits +- **Test Discovery**: Automatically discovers test bundles for each scenario +- **Platform Targeting**: Uses `TargetOS` property (ios, maccatalyst, android) +- **Queue Selection**: Platform-appropriate Helix queues +- **XHarness Integration**: Uses XHarness CLI for device orchestration + +### CustomCommands for Category Filtering + +When using category splitting, `CustomCommands` metadata overrides the default xharness invocation: + +```xml +xharness apple test --target "$target" --app "$app" --output-directory "$output_directory" --timeout "$timeout" --launch-timeout "$launch_timeout" --set-env="TestFilter=Category=CategoryName" +``` + +**Important**: Keep CustomCommands as a single line. Multi-line commands with `set -ex` can cause parse errors. + +## Troubleshooting + +### Common Issues + +1. **Build failures**: Ensure MSBuild tasks are built first +2. **Missing devices**: Check queue availability at [helix.dot.net](https://helix.dot.net) +3. **Authentication**: For CI, ensure proper Azure DevOps access tokens +4. **Timeouts**: Adjust `TestTimeout` and `WorkItemTimeout` for complex scenarios +5. **CustomCommands parse errors**: Keep commands on single line, avoid shell constructs like `set -ex` + +### Logging and Diagnostics + +- Use `/bl:filename.binlog` for detailed MSBuild logs +- Add `-verbosity:diag` for maximum diagnostic output +- Check Helix job results at the URL provided after submission +- Individual work item logs available at `https://helix.dot.net/api/2019-06-17/jobs/{jobId}/workitems/{workItemName}/console` + +## Additional Resources + +- [XHarness on Helix Documentation](https://github.com/dotnet/arcade/blob/main/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md) +- [Helix SDK Documentation](https://github.com/dotnet/arcade/blob/main/src/Microsoft.DotNet.Helix/Sdk/Readme.md) +- [Example Helix Run](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1115383&view=results) diff --git a/eng/helix_xharness.proj b/eng/helix_xharness.proj index 58ce1b2afbae..d7c0a6c5e42e 100644 --- a/eng/helix_xharness.proj +++ b/eng/helix_xharness.proj @@ -15,6 +15,18 @@ $(_MauiDotNetTfm) + + + + CollectionView;Shell;HybridWebView + + + + + $(RepoRoot)artifacts/bin/ @@ -90,18 +102,53 @@ <_MAUIScenarioSearch Include="@(MAUIScenario)" /> - + - + + + ios-simulator-64 + 02:00:00 + 01:15:00 + 00:10:00 + $([System.IO.Directory]::GetDirectories('$(ScenariosDir)Controls.DeviceTests/Release/$(TargetFrameworkToTest)-ios/', '*.app', System.IO.SearchOption.AllDirectories)) + xharness apple test --target "$target" --app "$app" --output-directory "$output_directory" --timeout "$timeout" --launch-timeout "$launch_timeout" --set-env="TestFilter=Category=%(Identity)" + + + + ios-simulator-64 + 02:00:00 + 01:15:00 + 00:10:00 + $([System.IO.Directory]::GetDirectories('$(ScenariosDir)Controls.DeviceTests/Release/$(TargetFrameworkToTest)-ios/', '*.app', System.IO.SearchOption.AllDirectories)) + xharness apple test --target "$target" --app "$app" --output-directory "$output_directory" --timeout "$timeout" --launch-timeout "$launch_timeout" --set-env="TestFilter=SkipCategories=$(ControlsTestCategoriesToSkipForRestOfTests)" + + + + ios-simulator-64 + 02:00:00 + 01:00:00 + + + + ios-simulator-64 + 02:00:00 + 01:00:00 + + + ios-simulator-64 + 02:00:00 + 01:00:00 + + ios-simulator-64 02:00:00 01:00:00 - %(_MAUIScenarioSearch.ScenarioDirectoryName) + - + maccatalyst 02:00:00 01:00:00 @@ -109,6 +156,7 @@ + <_apks Include="%(_MAUIScenarioSearch.PayloadDirectory)/Release/$(TargetFrameworkToTest)-android/**/*Signed.apk" /> @@ -122,6 +170,8 @@ $([System.String]::Copy('%(AndroidPackageName)').Replace('-Signed','')) + + diff --git a/src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs b/src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs index cc1ea7873a93..bc350fb68cca 100644 --- a/src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs +++ b/src/Core/tests/DeviceTests.Shared/DeviceTestSharedHelpers.cs @@ -28,14 +28,32 @@ public static List GetExcludedTestCategories([DynamicallyAccessedMembers #if IOS || MACCATALYST foreach (var en in Foundation.NSProcessInfo.ProcessInfo.Environment) { + string key = $"{en.Key}"; string filterValue = $"{en.Value}"; - if ($"{en.Key}" == "TestFilter" && filterValue.StartsWith("Category=")) + + if (key == "TestFilter") { - Console.WriteLine($"TestFilter: {filterValue}"); - string categoryToRun = $"{filterValue.Split('=')[1]}"; - var categories = new List(GetTestCategoryValues(testCategoryType)); - categories.Remove(categoryToRun); - return categories.Select(c => $"Category={c}").ToList(); + // Support TestFilter=Category=X (run only category X) + if (filterValue.StartsWith("Category=")) + { + Console.WriteLine($"TestFilter: {filterValue}"); + string categoryToRun = $"{filterValue.Split('=')[1]}"; + var categories = new List(GetTestCategoryValues(testCategoryType)); + categories.Remove(categoryToRun); + return categories.Select(c => $"Category={c}").ToList(); + } + + // Support TestFilter=SkipCategories=X,Y,Z (skip categories X, Y, Z) + if (filterValue.StartsWith("SkipCategories=")) + { + Console.WriteLine($"TestFilter: {filterValue}"); + var categoriesToSkip = filterValue.Substring("SkipCategories=".Length) + .Split(new[] { ',', ';' }) + .Select(c => c.Trim()) + .Where(c => !string.IsNullOrWhiteSpace(c)) + .ToList(); + return categoriesToSkip.Select(c => $"Category={c}").ToList(); + } } } #endif